Add support to create sparse serverless index method (#175)

rohanshah18 · rohanshah18 · commit 59b9f5f2178e · 2025-02-03T14:18:08.000-05:00
## Problem

Add support to create sparse serverless index method

## Solution

As a part of this PR, added following changes:
1. add createSparseServelessIndex() that doesn't accept dimension and
metric as compared to the traditional createServerlessIndex()
2. removed a validation to check for values (part of vector) being null
or empty at the time of upsert since values can be an empty array of
floats for sparse vector
3. metric is now optional for creating serverless index and will default
to cosine for dense index
4. added integration tests

## Type of Change

- [ ] Bug fix (non-breaking change which fixes an issue)
- [X] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- [ ] This change requires a documentation update
- [ ] Infrastructure change (CI configs, etc)
- [ ] Non-code change (docs, etc)
- [ ] None of the above: (explain here)

## Test Plan

Added integration tests
diff --git a/README.md b/README.md
@@ -162,7 +162,7 @@ Operations related to the building and managing of Pinecone indexes are called [
 You can use the Java SDK to create two types of indexes: [serverless indexes](https://docs.pinecone.io/guides/indexes/understanding-indexes#serverless-indexes) (recommended for most use cases) and 
 [pod-based indexes](https://docs.pinecone.io/guides/indexes/understanding-indexes#pod-based-indexes) (recommended for high-throughput use cases).
 
-### Create a serverless index
+### Create a dense serverless index
 
 The following is an example of creating a serverless index in the `us-west-2` region of AWS. For more information on 
 serverless and regional availability, see [Understanding indexes](https://docs.pinecone.io/guides/indexes/understanding-indexes#serverless-indexes).
@@ -187,6 +187,31 @@ tags.put("env", "test");
 IndexModel indexModel = pinecone.createServerlessIndex(indexName, similarityMetric, dimension, cloud, region, DeletionProtection.ENABLED, tags);
 ```
 
+### Create a sparse serverless index
+
+The following is an example of creating a sparse serverless index in the `us-east-1` region of AWS. For more information on
+serverless and regional availability, see [Understanding indexes](https://docs.pinecone.io/guides/indexes/sparse-indexes).
+
+```java
+import io.pinecone.clients.Pinecone;
+import org.openapitools.db_control.client.model.IndexModel;
+import org.openapitools.db_control.client.model.DeletionProtection;
+import java.util.HashMap;
+...
+
+Pinecone pinecone = new Pinecone.Builder("PINECONE_API_KEY").build();
+        
+String indexName = "example-index";
+int dimension = 1538;
+String cloud = "aws";
+String region = "us-east-1";
+HashMap<String, String> tags = new HashMap<>();
+tags.put("env", "test");
+String vectorType = "sparse";
+
+IndexModel indexModel = pinecone.createSparseServelessIndex(indexName, cloud, region, DeletionProtection.ENABLED, tags, vectorType);
+```
+
 ### Create a pod index
 
 The following is a minimal example of creating a pod-based index. For all the possible configuration options, see 
diff --git a/src/integration/java/io/pinecone/integration/controlPlane/serverless/SparseIndexTest.java b/src/integration/java/io/pinecone/integration/controlPlane/serverless/SparseIndexTest.java
@@ -0,0 +1,110 @@
+package io.pinecone.integration.controlPlane.serverless;
+
+import io.pinecone.clients.Index;
+import io.pinecone.clients.Pinecone;
+import io.pinecone.exceptions.PineconeNotFoundException;
+import io.pinecone.helpers.RandomStringBuilder;
+import io.pinecone.proto.UpsertResponse;
+import io.pinecone.unsigned_indices_model.QueryResponseWithUnsignedIndices;
+import org.junit.jupiter.api.*;
+import org.openapitools.db_control.client.model.DeletionProtection;
+import org.openapitools.db_control.client.model.IndexModel;
+
+import java.util.*;
+
+import static io.pinecone.helpers.TestUtilities.waitUntilIndexIsReady;
+import static org.junit.jupiter.api.Assertions.*;
+
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+public class SparseIndexTest {
+    static String indexName;
+    static Pinecone pinecone;
+
+    @BeforeAll
+    public static void setUp() throws InterruptedException {
+        indexName = RandomStringBuilder.build("sparse-index", 8);
+        pinecone = new Pinecone
+                .Builder(System.getenv("PINECONE_API_KEY"))
+                .withSourceTag("pinecone_test")
+                .build();
+    }
+
+    @Test
+    @Order(1)
+    public void createSparseIndex() {
+        Map<String, String> tags = new HashMap<>();
+        tags.put("env", "test");
+
+        // Create sparse Index
+        IndexModel indexModel = pinecone.createSparseServelessIndex(indexName,
+                "aws",
+                "us-east-1",
+                DeletionProtection.ENABLED,
+                tags,
+                "sparse");
+
+        assertNotNull(indexModel);
+        assertEquals(indexName, indexModel.getName());
+        assertEquals(IndexModel.MetricEnum.DOTPRODUCT, indexModel.getMetric());
+        assertEquals(indexModel.getDeletionProtection(), DeletionProtection.ENABLED);
+        assertEquals(indexModel.getTags(), tags);
+        assertEquals(indexModel.getVectorType(), "sparse");
+    }
+
+    @Test
+    @Order(2)
+    public void configureSparseIndex() throws InterruptedException {
+        String key = "flag";
+        String value = "internal";
+        Map<String, String> tags = new HashMap<>();
+        tags.put(key, value);
+
+        // Wait until index is ready
+        waitUntilIndexIsReady(pinecone, indexName, 200000);
+
+        // Disable deletion protection and add more index tags
+        pinecone.configureServerlessIndex(indexName, DeletionProtection.DISABLED, tags);
+        Thread.sleep(7000);
+
+        // Describe index to confirm deletion protection is disabled
+        IndexModel indexModel = pinecone.describeIndex(indexName);
+        assertEquals(indexModel.getDeletionProtection(), DeletionProtection.DISABLED);
+        assert indexModel.getTags() != null;
+        assertEquals(indexModel.getTags().get(key), value);
+    }
+
+    @Disabled
+    // @Order(3)
+    public void upsertAndQueryVectors() {
+        Index index = pinecone.getIndexConnection(indexName);
+        String id = "v1";
+        ArrayList<Long> indices = new ArrayList<>();
+        indices.add(1L);
+        indices.add(2L);
+
+        ArrayList<Float> values = new ArrayList<>();
+        values.add(1f);
+        values.add(2f);
+
+        UpsertResponse upsertResponse = index.upsert("v1", Collections.emptyList(), indices, values, null, "");
+        assertEquals(upsertResponse.getUpsertedCount(), 1);
+
+        // Query by vector id
+        QueryResponseWithUnsignedIndices queryResponse = index.queryByVectorId(1, id, true, false);
+        assertEquals(queryResponse.getMatchesList().size(), 1);
+        assertEquals(queryResponse.getMatches(0).getId(), id);
+        assertEquals(queryResponse.getMatches(0).getSparseValuesWithUnsignedIndices().getIndicesWithUnsigned32IntList(), indices);
+        assertEquals(queryResponse.getMatches(0).getSparseValuesWithUnsignedIndices().getValuesList(), values);
+    }
+
+    @Test
+    @Order(4)
+    public void deleteSparseIndex() throws InterruptedException {
+        // Delete sparse index
+        pinecone.deleteIndex(indexName);
+        Thread.sleep(5000);
+
+        // Confirm the index is deleted by calling describe index which should return resource not found
+        assertThrows(PineconeNotFoundException.class, () -> pinecone.describeIndex(indexName));
+    }
+}
diff --git a/src/integration/java/io/pinecone/integration/dataPlane/UpsertErrorTest.java b/src/integration/java/io/pinecone/integration/dataPlane/UpsertErrorTest.java
@@ -48,17 +48,7 @@ public void upsertWithApiKeyMissingSyncTest() {
             index.upsert(null, values);
             fail("Expecting invalid upsert request exception");
         } catch (PineconeException expected) {
-            assertEquals(expected.getMessage(), "Invalid upsert request. Please ensure that both id and values are provided.");
-        }
-    }
-
-    @Test
-    public void upsertWhenValuesMissingSyncTest() {
-        try {
-            index.upsert("some_id", null);
-            fail("Expecting invalid upsert request exception");
-        } catch (PineconeException expected) {
-            assertEquals(expected.getMessage(), "Invalid upsert request. Please ensure that both id and values are provided.");
+            assertEquals(expected.getMessage(), "Invalid upsert request. Please ensure that id is provided.");
         }
     }
 
@@ -126,17 +116,7 @@ public void upsertWithApiKeyMissingFutureTest() {
             asyncIndex.upsert(null, values);
             fail("Expecting invalid upsert request exception");
         } catch (PineconeException expected) {
-            assertTrue(expected.getMessage().contains("ensure that both id and values are provided."));
-        }
-    }
-
-    @Test
-    public void upsertWhenValuesMissingFutureTest() {
-        try {
-            asyncIndex.upsert("some_id", null);
-            fail("Expecting invalid upsert request exception");
-        } catch (PineconeException expected) {
-            assertTrue(expected.getMessage().contains("ensure that both id and values are provided."));
+            assertTrue(expected.getMessage().contains("ensure that id is provided."));
         }
     }
 
diff --git a/src/main/java/io/pinecone/clients/Pinecone.java b/src/main/java/io/pinecone/clients/Pinecone.java
@@ -87,8 +87,9 @@ public IndexModel createServerlessIndex(String indexName,
         }
 
         if (metric == null || metric.isEmpty()) {
-            throw new PineconeValidationException("Metric cannot be null or empty. Must be one of " + Arrays.toString(CreateIndexRequest.MetricEnum.values()));
+            metric = "cosine";
         }
+
         try {
             CreateIndexRequest.MetricEnum.fromValue(metric.toLowerCase());
         } catch (IllegalArgumentException e) {
@@ -142,6 +143,78 @@ public IndexModel createServerlessIndex(String indexName,
         return indexModel;
     }
 
+    /**
+     * Creates a new sparse serverless index.
+     * <p>
+     * Example:
+     * <pre>{@code
+     *     client.createServerlessIndex("YOUR-INDEX", "cosine", 1536, "aws", "us-west-2", DeletionProtection.ENABLED);
+     * }</pre>
+     *
+     * @param indexName The name of the index to be created.
+     * @param cloud The cloud provider for the index.
+     * @param region The cloud region for the index.
+     * @param deletionProtection Enable or disable deletion protection for the index.
+     * @param tags A map of tags to associate with the Index.
+     * @param vectorType The metric type for the index. Must be one of "cosine", "euclidean", or "dotproduct".
+     * @return {@link IndexModel} representing the created serverless index.
+     * @throws PineconeException if the API encounters an error during index creation or if any of the arguments are invalid.
+     */
+    public IndexModel createSparseServelessIndex(String indexName,
+                                                  String cloud,
+                                                  String region,
+                                                  DeletionProtection deletionProtection,
+                                                  Map<String, String> tags,
+                                                  String vectorType)  throws PineconeException {
+        if (indexName == null || indexName.isEmpty()) {
+            throw new PineconeValidationException("Index name cannot be null or empty");
+        }
+
+        if (cloud == null || cloud.isEmpty()) {
+            throw new PineconeValidationException("Cloud cannot be null or empty. Must be one of " + Arrays.toString(ServerlessSpec.CloudEnum.values()));
+        }
+
+        try {
+            ServerlessSpec.CloudEnum.fromValue(cloud.toLowerCase());
+        } catch (IllegalArgumentException e) {
+            throw new PineconeValidationException("Cloud cannot be null or empty. Must be one of " + Arrays.toString(ServerlessSpec.CloudEnum.values()));
+        }
+
+        if (region == null || region.isEmpty()) {
+            throw new PineconeValidationException("Region cannot be null or empty");
+        }
+
+        if(!vectorType.equalsIgnoreCase("sparse") && !vectorType.equalsIgnoreCase("dense")) {
+            throw new PineconeValidationException("vectorType must be sparse or dense");
+        }
+
+        // Convert user string for "cloud" arg into ServerlessSpec.CloudEnum
+        ServerlessSpec.CloudEnum cloudProvider = ServerlessSpec.CloudEnum.fromValue(cloud.toLowerCase());
+
+        ServerlessSpec serverlessSpec = new ServerlessSpec().cloud(cloudProvider).region(region);
+        IndexSpec createServerlessIndexRequestSpec = new IndexSpec().serverless(serverlessSpec);
+
+        IndexModel indexModel = null;
+
+        try {
+            CreateIndexRequest createIndexRequest = new CreateIndexRequest()
+                    .name(indexName)
+                    .metric(CreateIndexRequest.MetricEnum.DOTPRODUCT)
+                    .spec(createServerlessIndexRequestSpec)
+                    .deletionProtection(deletionProtection)
+                    .vectorType(vectorType);
+
+            if(tags != null && !tags.isEmpty()) {
+                createIndexRequest.tags(tags);
+            }
+
+            indexModel = manageIndexesApi.createIndex(createIndexRequest);
+        } catch (ApiException apiException) {
+            handleApiException(apiException);
+        }
+        return indexModel;
+    }
+
     /**
      * Overload for creating a new pods index with environment and podType, the minimum required parameters.
      * <p>
diff --git a/src/main/java/io/pinecone/commons/IndexInterface.java b/src/main/java/io/pinecone/commons/IndexInterface.java
@@ -103,12 +103,10 @@ default Vector buildUpsertVector(String id,
                                      List<Long> sparseIndices,
                                      List<Float> sparseValues,
                                      Struct metadata) {
-        if (id == null || id.isEmpty() || values == null || values.isEmpty()) {
-            throw new PineconeValidationException("Invalid upsert request. Please ensure that both id and values are " +
-                    "provided.");
+         if (id == null || id.isEmpty()) {
+            throw new PineconeValidationException("Invalid upsert request. Please ensure that id is provided.");
         }
 
-
         Vector.Builder vectorBuilder = Vector.newBuilder()
                 .setId(id)
                 .addAllValues(values);
diff --git a/src/test/java/io/pinecone/PineconeIndexOperationsTest.java b/src/test/java/io/pinecone/PineconeIndexOperationsTest.java
@@ -76,19 +76,6 @@ public void testCreateServerlessIndex() throws IOException {
                 () -> client.createServerlessIndex(null, "cosine", 3, "aws", "us-west-2", DeletionProtection.DISABLED, Collections.EMPTY_MAP));
         assertEquals("Index name cannot be null or empty", thrownNullIndexName.getMessage());
 
-        PineconeValidationException thrownEmptyMetric = assertThrows(PineconeValidationException.class,
-                () -> client.createServerlessIndex("testServerlessIndex", "", 3, "aws", "us-west-2", DeletionProtection.DISABLED, Collections.EMPTY_MAP));
-        assertEquals("Metric cannot be null or empty. Must be one of " + Arrays.toString(IndexModel.MetricEnum.values()), thrownEmptyMetric.getMessage());
-
-        PineconeValidationException thrownInvalidMetric = assertThrows(PineconeValidationException.class,
-                () -> client.createServerlessIndex("testServerlessIndex", "blah", 3, "aws", "us-west-2", DeletionProtection.DISABLED, Collections.EMPTY_MAP));
-        assertEquals(String.format("Metric cannot be null or empty. Must be one of " + Arrays.toString(IndexModel.MetricEnum.values())), thrownInvalidMetric.getMessage());
-
-        PineconeValidationException thrownNullMetric = assertThrows(PineconeValidationException.class,
-                () -> client.createServerlessIndex("testServerlessIndex", null, 3, "aws", "us-west-2", DeletionProtection.DISABLED, Collections.EMPTY_MAP));
-        assertEquals("Metric cannot be null or empty. Must be one of " + Arrays.toString(IndexModel.MetricEnum.values()),
-                thrownNullMetric.getMessage());
-
         PineconeValidationException thrownNegativeDimension = assertThrows(PineconeValidationException.class,
                 () -> client.createServerlessIndex("testServerlessIndex", "cosine", -3, "aws", "us-west-2", DeletionProtection.DISABLED, Collections.EMPTY_MAP));
         assertEquals("Dimension must be greater than 0. See limits for more info: https://docs.pinecone.io/reference/limits", thrownNegativeDimension.getMessage());
