3import { sqlToJSON } from "https://esm.town/v/nbbaier/sqliteExportHelpers?v=22";
4import { db as allValsDb } from "https://esm.town/v/sqlite/db?v=9";
5import OpenAI from "npm:openai";
6
7export default async function semanticSearchPublicVals(query) {
11 });
12
13 const openai = new OpenAI();
14
15 const embedding = await openai.embeddings.create({
16 model: "text-embedding-3-small",
17 input: query,
3Uses [Neon](https://neon.tech/) to search embeddings of all vals, using the [pg_vector](https://neon.tech/docs/extensions/pgvector) extension.
4
5- Call OpenAI to generate an embedding for the search query.
6- Query the `vals_embeddings` table in Neon using the cosine similarity operator.
7 - The `vals_embeddings` table gets refreshed every 10 minutes by [janpaul123/indexValsNeon](https://www.val.town/v/janpaul123/indexValsNeon).
1*Part of [Val Town Semantic Search](https://www.val.town/v/janpaul123/valtownsemanticsearch).*
2
3Generates OpenAI embeddings for all public vals, and stores them in [Neon](https://neon.tech/), using the [pg_vector](https://neon.tech/docs/extensions/pgvector) extension.
4
5- Create the `vals_embeddings` table in Neon if it doesn't already exist.
6- Get all val names from the [database of public vals](https://www.val.town/v/sqlite/db), made by [Achille Lacoin](https://www.val.town/u/pomdtr).
7- Get all val names from the `vals_embeddings` table and compute the difference (which ones are missing).
8- Iterate through all missing vals, get their code, get embeddings from OpenAI, and store the result in Neon.
9- Can now be searched using [janpaul123/semanticSearchNeon](https://www.val.town/v/janpaul123/semanticSearchNeon).
10
6import cosSimilarity from "npm:cos-similarity";
7import _ from "npm:lodash";
8import OpenAI from "npm:openai";
9
10const openai = new OpenAI();
11const queryEmbedding = (await openai.embeddings.create({
12 model: "text-embedding-3-small",
13 input: "check dynamicland website for changes and email me",
16console.log(queryEmbedding.slice(0, 4));
17
18const embedding = await openai.embeddings.create({
19 model: "text-embedding-3-small",
20 input: "check dynamicland website for changes and email me",
41}`;
42
43const queryEmbeddingVal = (await openai.embeddings.create({
44 model: "text-embedding-3-small",
45 input: valCode,
6 - The blobs have been generated by [janpaul123/indexValsBlobs](https://www.val.town/v/janpaul123/indexValsBlobs). It is not run automatically.
7- Get all blobs with embeddings pointed to by the metadata, e.g. `allValsBlob1536EmbeddingsData_0` for `batchDataIndex` 0.
8- Call OpenAI to generate an embedding for the search query.
9- Go through all embeddings and compute cosine similarity with the embedding for the search query.
10- Return list sorted by similarity.
6import { VectorDB } from "npm:imvectordb";
7import _ from "npm:lodash";
8import OpenAI from "npm:openai";
9
10export default async function semanticSearchPublicVals(query) {
34 }
35
36 const openai = new OpenAI();
37 const embedding = await openai.embeddings.create({
38 model: "text-embedding-3-small",
39 input: query,
1*Part of [Val Town Semantic Search](https://www.val.town/v/janpaul123/valtownsemanticsearch).*
2
3Generates OpenAI embeddings for all public vals, and stores them in Val Town's [blob storage](https://docs.val.town/std/blob/).
4
5- Create a new metadata object. Also has support for getting the previous metadata and only adding new vals, but that's currently disabled.
6- Get all val names from the [database of public vals](https://www.val.town/v/sqlite/db), made by [Achille Lacoin](https://www.val.town/u/pomdtr).
7- Put val names in batches. Vals in the same batch will have their embeddings stored in the same blob, at different offsets.
8- Iterate through all each batch, get code for all the vals, get embeddings from OpenAI, and store the result in a blob.
9- When finished, save the metadata JSON to its own blob.
10- Can now be searched using [janpaul123/semanticSearchBlobs](https://www.val.town/v/janpaul123/semanticSearchBlobs).
3import { sqlToJSON } from "https://esm.town/v/nbbaier/sqliteExportHelpers?v=22";
4import { db as allValsDb } from "https://esm.town/v/sqlite/db?v=9";
5import OpenAI from "npm:openai";
6import { truncateMessage } from "npm:openai-tokens";
7
8export default async function(interval: Interval) {
35 }
36
37 const openai = new OpenAI();
38 for (const val of newVals) {
39 const code = (await allValsDb.execute({
42 })).rows[0][0];
43
44 const embedding = await openai.embeddings.create({
45 model: "text-embedding-3-small",
46 input: truncateMessage(code, "text-embedding-3-small"),
3This val hosts an [HTTP server](https://janpaul123-valtownsemanticsearch.web.val.run/) that lets you search all vals based on vibes. If you search for "discord bot" it shows all vals that have "discord bot" vibes.
4
5It does this by comparing [embeddings from OpenAI](https://platform.openai.com/docs/guides/embeddings) generated for the code of all public vals, to an embedding of your search query.
6
7This is an experiment to see if and how we want to incorporate semantic search in the actual Val Town search page.
3Uses [Turso](https://turso.tech/) to search embeddings of all vals, using the [sqlite-vss](https://github.com/asg017/sqlite-vss) extension.
4
5- Call OpenAI to generate an embedding for the search query.
6- Query the `vss_vals_embeddings` table in Turso using `vss_search`.
7 - The `vss_vals_embeddings` table has been generated by [janpaul123/indexValsTurso](https://www.val.town/v/janpaul123/indexValsTurso). It is not run automatically.
