Skip to content

Commit

Permalink
Add debug logs on incremental facet indexing
Browse files Browse the repository at this point in the history
  • Loading branch information
ManyTheFish committed Feb 8, 2024
1 parent 801908f commit c659cc2
Show file tree
Hide file tree
Showing 2 changed files with 18 additions and 0 deletions.
17 changes: 17 additions & 0 deletions milli/src/update/facet/incremental.rs
Expand Up @@ -66,6 +66,7 @@ impl FacetsUpdateIncremental {
#[logging_timer::time("FacetsUpdateIncremental::{}")]
pub fn execute(self, wtxn: &mut RwTxn) -> crate::Result<()> {
let mut iter = self.delta_data.into_stream_merger_iter()?;
let mut current_field_id = None;

while let Some((key, value)) = iter.next()? {
if !valid_lmdb_key(key) {
Expand All @@ -75,6 +76,22 @@ impl FacetsUpdateIncremental {
let key = FacetGroupKeyCodec::<BytesRefCodec>::bytes_decode(key)
.map_err(heed::Error::Encoding)?;
let value = KvReader::new(value);

if current_field_id.map_or(true, |fid| fid != key.field_id) {
if let Some(fid) = current_field_id {
let highest_level = get_highest_level(wtxn, self.inner.db, fid)?;
let field_id_prefix = &fid.to_be_bytes();
let line_count = self
.inner
.db
.remap_types::<Bytes, DecodeIgnore>()
.prefix_iter(wtxn, field_id_prefix)?
.count();
log::debug!("field_id {fid} contains {line_count} keys and the highest level is {highest_level}");
}

current_field_id = Some(key.field_id)
}
let docids_to_delete = value
.get(DelAdd::Deletion)
.map(CboRoaringBitmapCodec::bytes_decode)
Expand Down
1 change: 1 addition & 0 deletions milli/src/update/facet/mod.rs
Expand Up @@ -150,6 +150,7 @@ impl<'i> FacetsUpdate<'i> {

// See self::comparison_bench::benchmark_facet_indexing
if self.data_size >= (self.database.len(wtxn)? / 50) {
debug!("Incremental indexing of {} keys", self.data_size);
let field_ids =
self.index.faceted_fields_ids(wtxn)?.iter().copied().collect::<Vec<_>>();
let bulk_update = FacetsUpdateBulk::new(
Expand Down

0 comments on commit c659cc2

Please sign in to comment.