I am ingesting 250 vectors. I do it in batches of 100. That is, 100, then 100, and then 50.
I know that I hit this code 3 times, but I see only 100 vectors when I login to the site…Sometimes I see some other numbers… Then I discover the some upsert calls throw exception: “Task has been canceled”…
Do I miss something here? Any ideas?
public async void IngestVectors(IndexClient<RestTransport> idx, List<ISegment> segList, string indexNamespace = null)
{
using (Log.VerboseCall())
{
if (idx == null)
throw new ArgumentNullException(nameof(idx));
if (idx.Index.Status.IsReady == false)
throw new Exception("Index is not in Ready state");
if (segList == null || segList.Count == 0)
throw new ArgumentException("Vector list is null or empty");
try
{
mIndex = idx;
List<Vector> vectors = new List<Vector>();
for (int i = 0; i < segList.Count; i++)
{
if (segList[i] == null || String.IsNullOrWhiteSpace(segList[i].SegmentText))
continue;
Vector vec = new Vector()
{
Id = i.ToString(),
Values = segList[i].TextVector,
Metadata = new MetadataMap()
};
vec.Metadata.Add("segmentId", segList[i].segmentId);
vec.Metadata.Add("SegmentText", segList[i].SegmentText);
if (!String.IsNullOrEmpty(segList[i].Source.Trim()))
vec.Metadata.Add("Source", segList[i].Source);
if (segList[i].SrcType >= 0)
vec.Metadata.Add("SrcType", segList[i].SrcType.ToString());
vec.Metadata.Add("End", segList[i].TimeIngested.ToString("yyyy/MM/dd HH:mm:ss"));
vectors.Add(vec);
}
await mIndex.Upsert(vectors, indexNamespace);
Log.VerboseFormat("Successfully ingested {0} vectors into Index: {1}", vectors.Count, mIndex.Index.Details.Name);
}
catch (Exception ex)
{
Log.Verbose(ex);
}
}
}