Skip to content

Commit

Permalink
Recreate Qdrant collection on delete, and wait for deletion to succeed (
Browse files Browse the repository at this point in the history
BloopAI#960)

* Wait max 60s before moving on from deleting the qdrant collection

* Nicer this way

* Immediately recreate the collection
  • Loading branch information
rsdy committed Sep 15, 2023
1 parent cb74634 commit 19947b7
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 3 deletions.
2 changes: 1 addition & 1 deletion server/bleep/src/indexes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ impl Indexes {
}

if let Some(ref semantic) = semantic {
semantic.delete_collection().await?;
semantic.reset_collection_blocking().await?;
}
}
config.source.save_index_version()?;
Expand Down
39 changes: 37 additions & 2 deletions server/bleep/src/semantic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ use std::{borrow::Cow, collections::HashMap, env, path::Path, sync::Arc};

use crate::{query::parser::SemanticQuery, Configuration};

use anyhow::bail;
use qdrant_client::{
prelude::{QdrantClient, QdrantClientConfig},
qdrant::{
Expand All @@ -15,7 +16,7 @@ use qdrant_client::{
use futures::{stream, StreamExt, TryStreamExt};
use rayon::prelude::*;
use thiserror::Error;
use tracing::{debug, info, warn};
use tracing::{debug, error, info, warn};

pub mod chunk;
pub mod embedder;
Expand Down Expand Up @@ -244,12 +245,46 @@ impl Semantic {
pub fn embedder(&self) -> &dyn Embedder {
self.embedder.as_ref()
}
pub async fn delete_collection(&self) -> anyhow::Result<()> {

pub async fn reset_collection_blocking(&self) -> anyhow::Result<()> {
_ = self
.qdrant
.delete_collection(&self.config.collection_name)
.await?;

let deleted = 'deleted: {
for _ in 0..60 {
match self
.qdrant
.has_collection(&self.config.collection_name)
.await
{
Ok(true) => {
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
}
Ok(false) => {
break 'deleted true;
}
Err(err) => {
error!(?err, "failed to delete qdrant collection for migration");
}
}
}
false
};

if !deleted {
error!("failed to delete qdrant collection after 60s");
bail!("deletion failed")
}

let CollectionOperationResponse { result, .. } =
create_collection(&self.config.collection_name, &self.qdrant)
.await
.unwrap();

assert!(result);

Ok(())
}

Expand Down

0 comments on commit 19947b7

Please sign in to comment.