Need help with Embeddings - TypeError: records.forEach is not a function

[nnettey]

I could use some help here, I’m using the following versions:

• NodeJS (v20.11.0)
• @pinecone-database/pinecone": “^3.0.2”
• OpenAI model text-embedding-ada-002

Error:

Error during embedding generation: TypeError: records.forEach is not a function

Code: embeddingController.js

const axios = require('axios');
const { Pinecone } = require('@pinecone-database/pinecone');
const { v4: uuidv4 } = require('uuid'); // Use UUID for unique IDs

(async () => {
  const fetch = (await import('node-fetch')).default;
  global.fetch = fetch; // Resolve fetch implementation warning
})();

// Initialize Pinecone client
const pinecone = new Pinecone({
    apiKey: process.env.PINECONE_API_KEY,
  });

  exports.generateEmbedding = async (req, res) => {
    const { text } = req.body;
  
    try {
      // Generate embedding from OpenAI API
      const response = await axios.post(
        'https://api.openai.com/v1/embeddings',
        {
          model: 'text-embedding-ada-002',
          input: text,
        },
        {
          headers: {
            Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
          },
        }
      );
  
      const embeddings = response.data.data;
      if (!Array.isArray(embeddings) || embeddings.length === 0) {
        throw new Error('Invalid embedding response from OpenAI API');
      }
  
      const embedding = embeddings[0].embedding;
      console.log('Length of the embedding:', embedding.length);
  
      // Check if the embedding dimension matches the required length for Pinecone
      if (embedding.length !== 1536) {
        throw new Error('Embedding dimension mismatch: Expected 1536.');
      }
  
      // Assign a unique ID to the vector
      const uniqueId = uuidv4();
  
      // Construct the vector to be upserted
      const vector = {
        id: uniqueId,
        values: embedding,
        metadata: { text: text }  // Store the original text as metadata
      };
  
      // Initialize the Pinecone index
      const index = pinecone.Index('secret');
  
      // Prepare the payload for upsert
      const upsertPayload = {
        vectors: [vector]  // Pass the vector in an array as required by Pinecone
      };
  
      // Log the payload before upserting
      console.log('Upsert payload:', upsertPayload);
  
      // Perform the upsert operation
      await index.upsert(upsertPayload);
  
      res.status(200).send({ message: 'Embedding generated and stored successfully.', id: uniqueId });
    } catch (error) {
      console.error('Error during embedding generation:', error);
      res.status(500).send(error.message);
    }
  };```

[ZacharyProser]

Hi @nnettey, and welcome to the Pinecone community forums!

Thank you for your question.

Based off your error message and the code you provided, it looks like the vectors you’re trying to upsert are malformed.

Could you please console.log your upsertPayload object prior to calling index.upsert? That will help us to see if something is wrong with your vector format.

At a high-level, your vectors need to look like this:

[
  { id: '1', values: [0.1, 0.2, 0.3] },
  { id: '2', values: [0.4, 0.5, 0.6] },
  // more records...
]

Hope this helps!
Zack

[nnettey]

Length of the embedding: 1536
First 5 values of embedding: [
-0.024920061,
-0.017879197,
-0.007459801,
-0.007932796,
-0.019419808
]
Generated uniqueId: d6745304-645e-41f9-bd3f-ddaa6169c497
Upsert payload (without full embedding): {
id: ‘d6745304-645e-41f9-bd3f-ddaa6169c497’,
values: ‘Array of length 1536’,
metadata: { text: ‘Downtown Store sales performance in January 2024.’ }
}

[nnettey]

I am just bringing more information (current code and error log) in hopes that it’ll provide more insights so someone can help me. Thanks in advance.
Code
const axios = require(‘axios’);
const { Pinecone } = require(‘@pinecone-database/pinecone’);
const { v4: uuidv4 } = require(‘uuid’);
require(‘dotenv’).config(); // Ensure that the .env file is loaded

// Initialize Pinecone client
let pinecone;

async function initializePinecone() {
console.log(‘Initializing Pinecone client’);
pinecone = new Pinecone({
apiKey: process.env.PINECONE_API_KEY, // Use the API key from the environment variable
});
console.log(‘Pinecone client initialized successfully’);
}

exports.generateEmbedding = async (req, res) => {
const { text } = req.body;

try {
console.log(‘Starting generateEmbedding function’);

// Initialize Pinecone if not already initialized
if (!pinecone) {
  await initializePinecone();
}

// Step 1: Obtain the embeddings from OpenAI
console.log('Step 1: Obtaining embeddings from OpenAI');
const openAIResponse = await getOpenAIEmbedding(text);
logOpenAIResponse(openAIResponse);

// Step 2: Inspect and analyze the OpenAI payload
console.log('Step 2: Inspecting OpenAI payload');
const embedding = inspectOpenAIPayload(openAIResponse);

// Step 3: Format the payload for Pinecone
console.log('Step 3: Formatting payload for Pinecone');
const pineconePayload = formatPineconePayload(embedding, text);
logPineconePayload(pineconePayload);

// Step 4: Upsert into Pinecone
console.log('Step 4: Upserting into Pinecone');
const upsertResponse = await upsertToPinecone(pineconePayload);
console.log('Pinecone Upsert Response:', JSON.stringify(upsertResponse, null, 2));

res.status(200).send({ message: 'Embedding generated and stored successfully.', id: pineconePayload.vectors[0].id });

} catch (error) {
console.error(‘Error during embedding generation:’, error);
res.status(500).send({
message: ‘Error during embedding generation’,
error: error.message,
stack: error.stack
});
}
};

async function getOpenAIEmbedding(text) {
console.log(‘Sending request to OpenAI API’);
try {
const response = await axios.post(
‘https://api.openai.com/v1/embeddings’,
{
model: ‘text-embedding-ada-002’,
input: text,
},
{
headers: {
Authorization: Bearer ${process.env.OPENAI_API_KEY}, // Use OpenAI API key from environment variable
},
}
);
console.log(‘Received response from OpenAI API’);
return response.data;
} catch (error) {
console.error(‘Error in getOpenAIEmbedding:’, error);
throw error;
}
}

function inspectOpenAIPayload(payload) {
console.log(‘Inspecting OpenAI payload’);
try {
if (!payload || !payload.data || !Array.isArray(payload.data) || payload.data.length === 0) {
throw new Error(‘Invalid response structure from OpenAI API’);
}

const embedding = payload.data[0].embedding;

if (!Array.isArray(embedding) || embedding.length === 0) {
  throw new Error('Invalid embedding in OpenAI API response');
}

if (embedding.length !== 1536) {
  throw new Error(`Embedding dimension mismatch: Expected 1536, got ${embedding.length}`);
}

console.log('OpenAI payload inspection completed successfully');
return embedding;

} catch (error) {
console.error(‘Error in inspectOpenAIPayload:’, error);
throw error;
}
}

function formatPineconePayload(embedding, text) {
console.log(‘Formatting Pinecone payload’);
try {
// Correct format for Pinecone upsert payload
return {
vectors: [
{
id: uuidv4(),
values: embedding,
metadata: { text: text }
}
]
};
} catch (error) {
console.error(‘Error in formatPineconePayload:’, error);
throw error;
}
}

async function upsertToPinecone(payload) {
console.log(‘Preparing to upsert to Pinecone’);
try {
const index = pinecone.index(process.env.PINECONE_INDEX);
console.log(‘Sending upsert request to Pinecone’);

// Ensure that payload.vectors is an array - Test Perplexity changes
if (!Array.isArray(payload.vectors)) {
    throw new Error('payload.vectors must be an array');
}
  
  // Restructure the payload to match Pinecone's expected format
  const vectors = payload.vectors.map(vector => ({
    id: vector.id,
    values: vector.values,
    metadata: vector.metadata
  }));

// Verify that vectors is an array - Test Perplexity changes
if (!Array.isArray(vectors)) {
    throw new Error('vectors must be an array');

//I added this from Perplexity - otherwise, everything seems to be just logging
    const vectors = Array.from(payload.vectors).map(vector => ({
    id: vector.id,
    values: vector.values,
    metadata: vector.metadata
    }));
}

  const response = await index.upsert({ vectors });
  console.log('Received upsert response from Pinecone');
  return response;
} catch (error) {
  console.error('Error in upsertToPinecone:', error);
  throw error;
}

}

// async function upsertToPinecone(payload) {
// console.log(‘Preparing to upsert to Pinecone’);
// try {
// const index = pinecone.index(process.env.PINECONE_INDEX); // Use the index name from the environment variable
// console.log(‘Sending upsert request to Pinecone’);
// const response = await index.upsert(payload); // Use the correct payload structure
// console.log(‘Received upsert response from Pinecone’);
// return response;
// } catch (error) {
// console.error(‘Error in upsertToPinecone:’, error);
// throw error;
// }
// }

function logOpenAIResponse(response) {
const { data, model, usage } = response;
console.log(‘OpenAI API Response:’);
console.log(‘Model:’, model);
console.log(‘Usage:’, usage);
console.log(‘Data:’);
data.forEach((item, index) => {
console.log( Item ${index + 1}:);
console.log(’ Object:‘, item.object);
console.log(’ Embedding:‘, item.embedding.slice(0, 5).map(n => n.toFixed(8)).join(’, ') + ‘…’);
console.log( (${item.embedding.length} total values));
});
}

function logPineconePayload(payload) {
console.log(‘Formatted Pinecone Payload:’);
payload.vectors.forEach((vector, index) => {
console.log( Vector ${index + 1}:);
console.log(’ ID:‘, vector.id);
console.log(’ Values:‘, vector.values.slice(0, 5).map(n => n.toFixed(8)).join(’, ‘) + ‘…’);
console.log( (${vector.values.length} total values));
console.log(’ Metadata:', JSON.stringify(vector.metadata));
});
}

Error:

smart-dashboard-backend@1.0.0 start
node server.js

Failed to find any user-provided fetch implementation. Using global fetch implementation.
Failed to find any user-provided fetch implementation. Using global fetch implementation.
Server running on http://localhost:3000
Starting generateEmbedding function
Initializing Pinecone client
Failed to find any user-provided fetch implementation. Using global fetch implementation.
Failed to find any user-provided fetch implementation. Using global fetch implementation.
Pinecone client initialized successfully
Step 1: Obtaining embeddings from OpenAI
Sending request to OpenAI API
Received response from OpenAI API
OpenAI API Response:
Model: text-embedding-ada-002
Usage: { prompt_tokens: 11, total_tokens: 11 }
Data:
Item 1:
Object: embedding
Embedding: -0.02492006, -0.01787920, -0.00745980, -0.00793280, -0.01941981…
(1536 total values)
Step 2: Inspecting OpenAI payload
Inspecting OpenAI payload
OpenAI payload inspection completed successfully
Step 3: Formatting payload for Pinecone
Formatting Pinecone payload
Formatted Pinecone Payload:
Vector 1:
ID: 3d96fdb4-59c4-4cf2-8f59-8b258f4f4455
Values: -0.02492006, -0.01787920, -0.00745980, -0.00793280, -0.01941981…
(1536 total values)
Metadata: {“text”:“Downtown Store sales performance in January 2024.”}
Step 4: Upserting into Pinecone
Preparing to upsert to Pinecone
Sending upsert request to Pinecone
Error in upsertToPinecone: TypeError: records.forEach is not a function
at UpsertCommand.validator (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:53:21)
at UpsertCommand. (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:71:30)
at step (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:33:23)
at Object.next (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:14:53)
at C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:8:71
at new Promise ()
at __awaiter (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:4:12)
at UpsertCommand.run (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:66:16)
at Index. (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\index.js:369:70)
at step (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\index.js:33:23)
Error during embedding generation: TypeError: records.forEach is not a function
at UpsertCommand.validator (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:53:21)
at UpsertCommand. (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:71:30)
at step (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:33:23)
at Object.next (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:14:53)
at C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:8:71
at new Promise ()
at __awaiter (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:4:12)
at step (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\index.js:33:23)
Error during embedding generation: TypeError: records.forEach is not a function
at UpsertCommand.validator (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:53:21)
at UpsertCommand. (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:71:30)
at step (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\index.js:33:23)
Error during embedding generation: TypeError: records.forEach is not a function
at UpsertCommand.validator (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:53:21)
at step (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\index.js:33:23)
at step (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\index.js:33:23)
at step (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\index.js:33:23)
Error during embedding generation: TypeError: records.forEach is not a function
at step (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\index.js:33:23)
Error during embedding generation: TypeError: records.forEach is not a function
at UpsertCommand.validator (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:53:21)
at UpsertCommand. (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:71:30)
at step (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:33:23)
at Object.next (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:14:53)
at C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:8:71
at step (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\index.js:33:23)
Error during embedding generation: TypeError: records.forEach is not a function
at UpsertCommand.validator (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:53:21)
at UpsertCommand. (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:71:30)
at step (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:33:23)
at step (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\index.js:33:23)
Error during embedding generation: TypeError: records.forEach is not a function
at UpsertCommand.validator (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:53:21)
at step (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\index.js:33:23)
at step (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\index.js:33:23)
Error during embedding generation: TypeError: records.forEach is not a function
at UpsertCommand.validator (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:53:21)
at UpsertCommand. (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:71:30)
at step (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\index.js:33:23)
Error during embedding generation: TypeError: records.forEach is not a function
at UpsertCommand.validator (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:53:21)
at step (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\index.js:33:23)
Error during embedding generation: TypeError: records.forEach is not a function
at UpsertCommand.validator (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:53:21)
at step (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\index.js:33:23)
Error during embedding generation: TypeError: records.forEach is not a function
at step (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\index.js:33:23)
at step (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\index.js:33:23)
Error during embedding generation: TypeError: records.forEach is not a function
at step (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\index.js:33:23)
at step (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\index.js:33:23)
at step (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\index.js:33:23)
Error during embedding generation: TypeError: records.forEach is not a function
at UpsertCommand.validator (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:53:21)
at UpsertCommand. (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:71:30)
at step (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:33:23)
at Object.next (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:14:53)
at C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:8:71
at new Promise ()
at __awaiter (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:4:12)
at UpsertCommand.run (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\upsert.js:66:16)
at Index. (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\index.js:369:70)
at step (C:\Data\LocalNettCreative\smart-dashboard-backend\node_modules@pinecone-database\pinecone\dist\data\index.js:33:23)

[ZacharyProser]

@nnettey

Length of the embedding: 1536
First 5 values of embedding: [
-0.024920061,
-0.017879197,
-0.007459801,
-0.007932796,
-0.019419808
]
Generated uniqueId: d6745304-645e-41f9-bd3f-ddaa6169c497
Upsert payload (without full embedding): {
id: ‘d6745304-645e-41f9-bd3f-ddaa6169c497’,
values: ‘Array of length 1536’,
metadata: { text: ‘Downtown Store sales performance in January 2024.’ }
}

Hi @nnettey, just to confirm, the values field actually contains an array like [-0.024920061, -0.017879197...], correct?

[nnettey]

Yes…and I truncated the list for brevity.

[appyou.labs]

Any solution to this “TypeError: records.forEach is not a function” while upserting.
I too facing this issue

[appyou.labs]

This code solved my issue:

await index.upsert([   
          {
            id: id,
            values: embedding,
            metadata: { text: text }
          }             
    ]);

[brian2]

This is still a problem and the solution by @appyou.labs is the answer.