# Even ChaGPT is unable to get past these errors. what should be changed

**URL:** <https://community.pinecone.io/t/even-chagpt-is-unable-to-get-past-these-errors-what-should-be-changed/1046>\
**Category:** Support\
**Created:** [April 5, 2023, 5:29pm UTC](https://community.pinecone.io/t/even-chagpt-is-unable-to-get-past-these-errors-what-should-be-changed/1046 "2023-04-05T17:29:37Z")\
**Posts on this page:** 4\
**Page:** 1

<div class="post-metadata">

**Author:** ![VishwasAI](https://avatars.discourse-cdn.com/v4/letter/v/47e85d/32.png) [@VishwasAI](https://community.pinecone.io/u/VishwasAI)\
**Post date:** [April 5, 2023, 5:29pm UTC](https://community.pinecone.io/t/even-chagpt-is-unable-to-get-past-these-errors-what-should-be-changed/1046/1 "2023-04-05T17:29:37Z")

</div>

def get\_embedding(text, model=“text-embedding-ada-002”):  
text = text.replace(“\n”, " “)  
try:  
result = openai.Embedding.create(input=[text], model=model)  
if ‘data’ in result and result[‘data’] and ‘embedding’ in result[‘data’][0]:  
return result[‘data’][0][‘embedding’]  
else:  
print(f"Error: Invalid embedding result for text: {text}”)  
return None  
except Exception as e:  
print(f"Error: {e} occurred while creating embedding for text: {text}")  
return None

import ast

# Generate embeddings

df\_new[‘ada\_embedding’] = df\_new[‘ada\_embedding’].apply(lambda x: ast.literal\_eval(x) if isinstance(x, str) else x)

# Filter out rows with None values in the ada\_embedding column

#df\_new = df\_new[df\_new[‘ada\_embedding’].notna()]

# Ensure embeddings are in the correct format (e.g., lists or arrays of numerical values)

df\_new[‘ada\_embedding’] = df\_new[‘ada\_embedding’].apply(lambda x: list(x) if isinstance(x, (list, np.ndarray)) else None)

# Create an index

index = pinecone.Index(“apex-reference”)

from tqdm.auto import tqdm

import numpy as np  
import json

def to\_vectors(embeddings):  
vectors =   
for embedding in embeddings:  
try:  
if isinstance(embedding, str):  
vector = np.array(ast.literal\_eval(embedding))  
elif isinstance(embedding, list):  
vector = np.array(embedding)  
else:  
raise TypeError(“Invalid input type”)  
vectors.append(vector)  
except (ValueError, TypeError, SyntaxError):  
print(f"Could not convert embedding {embedding} to vector")  
continue  
return vectors

# we will use batches of 5

batch\_size = 1

for i in tqdm(range(0, len(df\_new), batch\_size)):  
# Find end of batch  
i\_end = min(i + batch\_size, len(df\_new))  
# Extract batch  
batch = df\_new.iloc[i:i\_end]  
# Generate embeddings for batch (ensure embeddings are in the correct format)  
emb = to\_vectors(batch[‘ada\_embedding’])  
# Get metadata (optional)  
meta = batch[[‘PageNumber’, ‘PageContent’]].to\_dict(orient=‘records’)  
# Create unique IDs  
ids = [f"{idx}" for idx in range(i, i\_end)]  
# Create a dictionary for upsert  
to\_upsert = dict(zip(ids, emb))  
# Upsert/insert these records to Pinecone  
if not to\_upsert:  
print(“Empty batch, skipping…”)  
continue  
try:  
\_ = index.upsert(vectors=to\_upsert, metadata=meta)  
except Exception as e:  
print(f"Error: {e} occurred during upsert, skipping batch…")  
continue

# check that we have all vectors in index

index.describe\_index\_stats()

Error: Error: Invalid vector value passed: cannot interpret type \<class ‘str’\> occurred during upsert, skipping batch…

---

<div class="post-metadata">

**Author:** ![VishwasAI](https://avatars.discourse-cdn.com/v4/letter/v/47e85d/32.png) [@VishwasAI](https://community.pinecone.io/u/VishwasAI)\
**Post date:** [April 6, 2023, 3:51am UTC](https://community.pinecone.io/t/even-chagpt-is-unable-to-get-past-these-errors-what-should-be-changed/1046/2 "2023-04-06T03:51:58Z")

</div>

I tried this which works  
\_ = index.upsert(vectors=[  
(f"Page\_{i}", embeds[0],{})  
])

Not sure whats issue with zip

---

<div class="post-metadata">

**Author:** ![VishwasAI](https://avatars.discourse-cdn.com/v4/letter/v/47e85d/32.png) [@VishwasAI](https://community.pinecone.io/u/VishwasAI)\
**Post date:** [April 6, 2023, 3:54am UTC](https://community.pinecone.io/t/even-chagpt-is-unable-to-get-past-these-errors-what-should-be-changed/1046/3 "2023-04-06T03:54:29Z")

</div>

Revised code for reference. Note that to\_upsert zip does not work but some low level format works. Can some expert from pinecone support please help? Is this bug?

# we will use batches of 5

batch\_size = 1

for i in tqdm(range(0, len(df\_new), batch\_size)):  
# Find end of batch  
i\_end = min(i + batch\_size, len(df\_new))  
# Extract batch  
batch = df\_new.iloc[i:i\_end]  
# Generate embeddings for batch (ensure embeddings are in the correct format)  
res = openai.Embedding.create(input=list(batch[‘combined’]), engine=‘text-embedding-ada-002’)  
embeds = [record[‘embedding’] for record in res[‘data’]]  
# Get metadata (optional)  
meta = batch[[‘PageNumber’, ‘PageContent’]].to\_dict(orient=‘records’)  
# Create unique IDs  
ids = [f"Page\_{idx}" for idx in range(i, i\_end)]  
# Create a dictionary for upsert  
to\_upsert = dict(zip(ids, embeds))  
#print(to\_upsert)  
# Upsert/insert these records to Pinecone  
if not to\_upsert:  
print(“Empty batch, skipping…”)  
continue  
try:  
print(i,embeds)  
\_ = index.upsert(vectors=[  
(f"Page\_{i}“, embeds[0],{})  
])  
except Exception as e:  
print(f"Error: {e} occurred during upsert, skipping batch…”)  
continue

# check that we have all vectors in index

index.describe\_index\_stats()

---

<div class="post-metadata">

**Author:** ![VishwasAI](https://avatars.discourse-cdn.com/v4/letter/v/47e85d/32.png) [@VishwasAI](https://community.pinecone.io/u/VishwasAI)\
**Post date:** [April 6, 2023, 2:04pm UTC](https://community.pinecone.io/t/even-chagpt-is-unable-to-get-past-these-errors-what-should-be-changed/1046/4 "2023-04-06T14:04:27Z")

</div>

Ok so finally this worked. Hopefully it will save countless hours of efforts for someone else

index = pinecone.Index(“apex-reference”)

from tqdm.auto import tqdm

import numpy as np

# we will use batches of 64

batch\_size = 64

for i in tqdm(range(0, len(df\_new), batch\_size)):  
# Find end of batch  
i\_end = min(i + batch\_size, len(df\_new))  
# Extract batch  
batch = df\_new.iloc[i:i\_end]  
# Generate embeddings for batch (ensure embeddings are in the correct format)  
res = openai.Embedding.create(input=list(batch[‘combined’]), engine=‘text-embedding-ada-002’)  
embeds = [record[‘embedding’] for record in res[‘data’]]  
# Get metadata (optional)  
meta = batch[[‘PageNumber’, ‘PageContent’]].to\_dict(orient=‘records’)  
# Create unique IDs  
ids = [f"Page\_{idx}" for idx in range(i, i\_end)]  
# Create a dictionary for upsert  
to\_upsert = dict(zip(ids, embeds))  
# Convert the to\_upsert dictionary to the expected format for Pinecone  
to\_upsert\_list = [(k, v, m) for k, v, m in zip(to\_upsert.keys(), to\_upsert.values(), meta)]  
#print(to\_upsert)  
# Upsert/insert these records to Pinecone  
if not to\_upsert\_list:  
print(“Empty batch, skipping…”)  
continue  
try:  
print(i, embeds)  
\_ = index.upsert(vectors=to\_upsert\_list)  
except Exception as e:  
print(f"Error: {e} occurred during upsert, skipping batch…")  
continue