feature: readd recency bias

clients/ts-sdk/src/__tests__/constants.ts

@@ -8,11 +8,11 @@ export const TRIEVE = new TrieveSDK({
   organizationId: "de73679c-707f-4fc2-853e-994c910d944c",
 });
 
-// export const TRIEVELOCAL = new TrieveSDK({
+// export const TRIEVE = new TrieveSDK({
 //   baseUrl: "http://localhost:8090",
-//   organizationId: "f8bd8fc0-0f66-48d3-92be-a2e5f74c7952",
-//   datasetId: "506effe8-c460-4ec0-8576-6dd6ef1a57f0",
-//   apiKey: "tr-x5I18LghGCL22qXYQS6BrlJbqAOPI7Tz",
+//   organizationId: "967d4740-d8f0-4f3a-8a62-3c1297e5f6c4",
+//   datasetId: "88fb2a53-17bd-4311-9763-051dc5c9c476",
+//   apiKey: "tr-5OiU6tPsjgcMz0AeujPbKlBJFqeXVJ9G",
 // });
 
 export const EXAMPLE_TOPIC_ID = "f85984e1-7818-4971-b300-2f462fe1a5a2";

clients/ts-sdk/src/functions/organization/organization.test.ts

@@ -3,132 +3,141 @@ import { TRIEVE } from "../../__tests__/constants";
 import { TrieveSDK } from "../../sdk";
 import { CreateApiKeyResponse, ReturnQueuedChunk } from "../../types.gen";
 
-describe("Organization Tests", async () => {
-  let trieve: TrieveSDK;
-  beforeAll(() => {
-    trieve = TRIEVE;
-  });
-
-  test("create an api key and verify it works", async () => {
-    const apiKeyResponse = await trieve.createOrganizationApiKey({
-      role: 1,
-      name: "test suite key",
+describe(
+  "Organization Tests",
+  async () => {
+    let trieve: TrieveSDK;
+    beforeAll(() => {
+      trieve = TRIEVE;
     });
 
-    expectTypeOf(apiKeyResponse).toEqualTypeOf<CreateApiKeyResponse>();
+    test("create an api key and verify it works", async () => {
+      const apiKeyResponse = await trieve.createOrganizationApiKey({
+        role: 1,
+        name: "test suite key",
+      });
 
-    const newTrieve = new TrieveSDK({
-      apiKey: apiKeyResponse.api_key,
-      datasetId: trieve.datasetId,
-    });
+      expectTypeOf(apiKeyResponse).toEqualTypeOf<CreateApiKeyResponse>();
 
-    const queuedChunk = await newTrieve.createChunk({
-      chunk_html: "testing hello world",
-      tracking_id: "1234",
-      tag_set: ["test"],
-    });
+      const newTrieve = new TrieveSDK({
+        apiKey: apiKeyResponse.api_key,
+        datasetId: trieve.datasetId,
+        baseUrl: trieve.trieve.baseUrl,
+      });
 
-    expectTypeOf(queuedChunk).toEqualTypeOf<ReturnQueuedChunk>();
+      const queuedChunk = await newTrieve.createChunk({
+        chunk_html: "testing hello world",
+        tracking_id: "1234",
+        tag_set: ["test"],
+      });
 
-    newTrieve.deleteChunkByTrackingId({
-      trackingId: "1234",
-    });
-  });
-
-  test("create an expired api key and verify it does not work", async () => {
-    const apiKeyResponse = await trieve.createOrganizationApiKey({
-      expires_at: new Date(new Date().setDate(new Date().getDate() - 1))
-        .toISOString()
-        .slice(0, 19)
-        .replace("T", " "),
-      role: 1,
-      name: "test suite key",
+      expectTypeOf(queuedChunk).toEqualTypeOf<ReturnQueuedChunk>();
+
+      newTrieve.deleteChunkByTrackingId({
+        trackingId: "1234",
+      });
     });
 
-    expectTypeOf(apiKeyResponse).toEqualTypeOf<CreateApiKeyResponse>();
+    test("create an expired api key and verify it does not work", async () => {
+      const apiKeyResponse = await trieve.createOrganizationApiKey({
+        expires_at: new Date(new Date().setDate(new Date().getDate() - 1))
+          .toISOString()
+          .slice(0, 19)
+          .replace("T", " "),
+        role: 1,
+        name: "test suite key",
+      });
+
+      expectTypeOf(apiKeyResponse).toEqualTypeOf<CreateApiKeyResponse>();
+      console.log(apiKeyResponse);
+
+      let errorOccurred = false;
 
-    let errorOccurred = false;
+      const newTrieve = new TrieveSDK({
+        apiKey: apiKeyResponse.api_key,
+        datasetId: trieve.datasetId,
+        baseUrl: trieve.trieve.baseUrl,
+      });
 
-    const newTrieve = new TrieveSDK({
-      apiKey: apiKeyResponse.api_key,
-      datasetId: trieve.datasetId,
+      try {
+        await newTrieve.createChunk({
+          chunk_html: "testing hello world",
+          tracking_id: "should_never_work",
+          tag_set: ["test"],
+        });
+
+        newTrieve.deleteChunkByTrackingId({
+          trackingId: "should_never_work",
+        });
+        console.log("should not have worked");
+      } catch (e) {
+        errorOccurred = true;
+      }
+
+      expect(errorOccurred).toBe(true);
     });
 
-    try {
-      await newTrieve.createChunk({
-        chunk_html: "testing hello world",
-        tracking_id: "should_never_work",
-        tag_set: ["test"],
+    test("create an api key with a filter for test and verify it excludes chunks without the tag", async () => {
+      const apiKeyResponse = await trieve.createOrganizationApiKey({
+        role: 1,
+        name: "test suite key",
+        default_params: {
+          filters: {
+            must: [
+              {
+                field: "tag_set",
+                match_all: ["test"],
+              },
+            ],
+          },
+        },
       });
 
-      newTrieve.deleteChunkByTrackingId({
-        trackingId: "should_never_work",
+      expectTypeOf(apiKeyResponse).toEqualTypeOf<CreateApiKeyResponse>();
+
+      const newTrieve = new TrieveSDK({
+        apiKey: apiKeyResponse.api_key,
+        datasetId: trieve.datasetId,
+        baseUrl: trieve.trieve.baseUrl,
       });
-    } catch (e) {
-      errorOccurred = true;
-    }
-
-    expect(errorOccurred).toBe(true);
-  });
-
-  test("create an api key with a filter for test and verify it excludes chunks without the tag", async () => {
-    const apiKeyResponse = await trieve.createOrganizationApiKey({
-      role: 1,
-      name: "test suite key",
-      default_params: {
+
+      const queuedChunks = await newTrieve.createChunk([
+        {
+          chunk_html: "testing hello world",
+          tracking_id: "not_test",
+          tag_set: ["not_test"],
+        },
+        {
+          chunk_html: "testing hello world",
+          tracking_id: "test",
+          tag_set: ["test"],
+        },
+      ]);
+
+      expectTypeOf(queuedChunks).toEqualTypeOf<ReturnQueuedChunk>();
+
+      await new Promise((r) => setTimeout(r, 10000));
+
+      const chunksResp = await newTrieve.scroll({
+        page_size: 100,
         filters: {
           must: [
             {
               field: "tag_set",
-              match_all: ["test"],
+              match_all: ["not_test"],
             },
           ],
         },
-      },
-    });
-
-    expectTypeOf(apiKeyResponse).toEqualTypeOf<CreateApiKeyResponse>();
-
-    const newTrieve = new TrieveSDK({
-      apiKey: apiKeyResponse.api_key,
-      datasetId: trieve.datasetId,
-    });
-
-    const queuedChunks = await newTrieve.createChunk([
-      {
-        chunk_html: "testing hello world",
-        tracking_id: "not_test",
-        tag_set: ["not_test"],
-      },
-      {
-        chunk_html: "testing hello world",
-        tracking_id: "test",
-        tag_set: ["test"],
-      },
-    ]);
-
-    expectTypeOf(queuedChunks).toEqualTypeOf<ReturnQueuedChunk>();
-
-    await new Promise((r) => setTimeout(r, 10000));
-
-    const chunksResp = await newTrieve.scroll({
-      page_size: 100,
-      filters: {
-        must: [
-          {
-            field: "tag_set",
-            match_all: ["not_test"],
-          },
-        ],
-      },
-    });
+      });
 
-    for (const chunk of chunksResp.chunks) {
-      expect(chunk.tag_set).toContain("test");
-    }
+      for (const chunk of chunksResp.chunks) {
+        expect(chunk.tag_set).toContain("test");
+      }
 
-    newTrieve.deleteChunkByTrackingId({
-      trackingId: "1234",
+      newTrieve.deleteChunkByTrackingId({
+        trackingId: "1234",
+      });
     });
-  });
-});
+  },
+  { timeout: 100000 },
+);

server/migrations/2024-11-28-041452_add_on_delete_cascade_for_org_api_key/down.sql

@@ -0,0 +1,5 @@
+-- This file should undo anything in `up.sql`
+ALTER TABLE organization_api_key
+	DROP CONSTRAINT organization_api_key_organization_id_fkey,
+	ADD CONSTRAINT organization_api_key_organization_id_fkey
+	FOREIGN KEY (organization_id) REFERENCES organizations(id);

server/migrations/2024-11-28-041452_add_on_delete_cascade_for_org_api_key/up.sql

@@ -0,0 +1,5 @@
+-- Your SQL goes here
+ALTER TABLE organization_api_key
+	DROP CONSTRAINT organization_api_key_organization_id_fkey,
+	ADD CONSTRAINT organization_api_key_organization_id_fkey
+	FOREIGN KEY (organization_id) REFERENCES organizations(id) ON DELETE CASCADE;

server/src/data/models.rs

@@ -6661,6 +6661,8 @@ pub struct SortOptions {
     pub sort_by: Option<QdrantSortBy>,
     /// Location lets you rank your results by distance from a location. If not specified, this has no effect. Bias allows you to determine how much of an effect the location of chunks will have on the search results. If not specified, this defaults to 0.0. We recommend setting this to 1.0 for a gentle reranking of the results, >3.0 for a strong reranking of the results.
     pub location_bias: Option<GeoInfoWithBias>,
+    /// Recency Bias lets you determine how much of an effect the recency of chunks will have on the search results. If not specified, this defaults to 0.0. We recommend setting this to 1.0 for a gentle reranking of the results, >3.0 for a strong reranking of the results.
+    pub recency_bias: Option<f32>,
     /// Set use_weights to true to use the weights of the chunks in the result set in order to sort them. If not specified, this defaults to true.
     pub use_weights: Option<bool>,
     /// Tag weights is a JSON object which can be used to boost the ranking of chunks with certain tags. This is useful for when you want to be able to bias towards chunks with a certain tag on the fly. The keys are the tag names and the values are the weights.

server/src/operators/organization_operator.rs

@@ -777,6 +777,11 @@ pub async fn get_assumed_user_by_organization_api_key(
 
     let api_key: OrganizationApiKey = organization_api_key_columns::organization_api_key
         .filter(organization_api_key_columns::api_key_hash.eq(hash_function(api_key)))
+        .filter(
+            organization_api_key_columns::expires_at
+                .is_null()
+                .or(organization_api_key_columns::expires_at.ge(diesel::dsl::now.nullable())),
+        )
         .select(OrganizationApiKey::as_select())
         .first::<OrganizationApiKey>(&mut conn)
         .await