1import { extractValInfo } from "https://esm.town/v/pomdtr/extractValInfo";
2import { ComputeJSON, ComputeText, GenerateImage, sb, StableVideoDiffusion, Substrate } from "npm:substrate";
3import { z } from "npm:zod";
4import { zodToJsonSchema } from "npm:zod-to-json-schema";
13const sentencesSchema = z.object({
14 sentences: z.array(z.string()).length(5).describe("5 short sentences condensing the story"),
15 imageDescriptions: z
16 .array(z.string())
17 .length(5)
18 .describe(
19 "5 descriptions of images to accompany each sentence. Describe simple images that depict the setting of each scene. Do not include characters, just the background. Return just the description, no prefix or explanation.",
20 ),
21});
22const sentences = new ComputeJSON({
23 prompt: sb
24 .interpolate`We're making a storyboard. Generate 5 short sentences condensing the story about ${topic}, along with descriptions of images to accompany each sentence.`,
25 json_schema: zodToJsonSchema(sentencesSchema),
26});
28let videoNodes = [];
29for (let i = 0; i < 5; i++) {
30 const image = new GenerateImage({
31 prompt: sentences.future.json_object.get("imageDescriptions").at(i),
32 store: "hosted",
33 });
34 image.cache_age = 60 * 60 * 24 * 7;
35 const video = new StableVideoDiffusion({
36 image_uri: image.future.image_uri,
37 store: "hosted",
38 motion_bucket_id: 20,
80 "The roar of the falls subsides as the traveler rounds a bend in the path.",
81 ],
82 "imageDescriptions": [
83 "A dense jungle with towering trees and vines",
84 "A misty forest with ferns and moss-covered tree trunks",
7* Type text prompts, select it, press "Q". Select a previous generation with a new text prompt to keep iterating. Selecting shapes doesn't work yet. Have fun!
8
9<a href="https://x.com/JanPaul123/status/1815502582015754657"><img width=500 src="https://imagedelivery.net/iHX6Ovru0O7AjmyT5yZRoA/5893dfbf-c2de-4be0-049e-d8fdd1970a00/public"/></a>
10
7* Type text prompts, select it, press "Q". Select a previous generation with a new text prompt to keep iterating. Selecting shapes doesn't work yet. Have fun!
8
9<a href="https://x.com/JanPaul123/status/1815502582015754657"><img width=500 src="https://imagedelivery.net/iHX6Ovru0O7AjmyT5yZRoA/5893dfbf-c2de-4be0-049e-d8fdd1970a00/public"/></a>
10
10* Create a [Val Town API token](https://www.val.town/settings/api), open the browser preview of this val, and use the API token as the password to log in.
11
12<img width=500 src="https://imagedelivery.net/iHX6Ovru0O7AjmyT5yZRoA/7077d1b5-1fa7-4a9b-4b93-f8d01d3e4f00/public"/>
39 functions where possible. Unless specified, don't add error handling,
40 make sure that errors bubble up to the caller.
41 Avoid external images or base64 images, use emojis, unicode symtols, or icon fonts/libraries instead, unless that's
42 not practical for the user's request (e.g. if they ask for a particular animated gif).
43 If the user asks for something that requires persistence, use the Val Town Blob storage API, unless
42 functions where possible. Unless specified, don't add error handling,
43 make sure that errors bubble up to the caller.
44 Avoid external images or base64 images, use emojis, unicode symtols, or icon fonts/libraries instead, unless that's
45 not practical for the user's request (e.g. if they ask for a particular animated gif).
46 If the user asks for something that requires persistence, use the Val Town Blob storage API, unless
42 functions where possible. Unless specified, don't add error handling,
43 make sure that errors bubble up to the caller.
44 Avoid external images or base64 images, use emojis, unicode symtols, or icon fonts/libraries instead, unless that's
45 not practical for the user's request (e.g. if they ask for a particular animated gif).
46 If the user asks for something that requires persistence, use the Val Town Blob storage API, unless
101 <meta charset="UTF-8" />
102 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
103 <link rel="icon" href="https://www.substrate.run/favicon.ico" type="image/x-icon">
104 <link
105 href="https://fonts.googleapis.com/css2?family=Roboto:wght@700&display=swap"
1#### SHOTCLIP
2
3Demo of embedding images with [substrate](https://www.substrate.run/), and querying them for semantic relevance.
4
5Use the query parameter `prompt` to control the search.
10* Create a [Val Town API token](https://www.val.town/settings/api), open the browser preview of this val, and use the API token as the password to log in.
11
12<img width=500 src="https://imagedelivery.net/iHX6Ovru0O7AjmyT5yZRoA/7077d1b5-1fa7-4a9b-4b93-f8d01d3e4f00/public"/>
