fix hnsw deadlock by adding cleanup interval/dropping shard

adapters/repos/db/helper_for_test.go

@@ -200,7 +200,7 @@ func testShard(ctx context.Context) *Shard {
 	idx := &Index{
 		Config:                IndexConfig{RootPath: dirName},
 		invertedIndexConfig:   &models.InvertedIndexConfig{CleanupIntervalSeconds: 1},
-		vectorIndexUserConfig: hnsw.UserConfig{},
+		vectorIndexUserConfig: hnsw.UserConfig{CleanupIntervalSeconds: 1},
 		logger:                logrus.New(),
 		getSchema:             schemaGetter,
 	}

adapters/repos/db/index.go

@@ -871,7 +871,7 @@ func (i *Index) drop() error {
 			// shouldn't
 			continue
 		}
-		err := shard.drop()
+		err := shard.drop(false)
 		if err != nil {
 			return errors.Wrapf(err, "delete shard %s", shard.ID())
 		}

adapters/repos/db/shard.go

@@ -168,8 +168,8 @@ func (s *Shard) initDBFile(ctx context.Context) error {
 	return nil
 }
 
-func (s *Shard) drop() error {
-	if s.isReadOnly() {
+func (s *Shard) drop(force bool) error {
+	if s.isReadOnly() && !force {
 		return storagestate.ErrStatusReadOnly
 	}
 

adapters/repos/db/shard_status.go

@@ -18,13 +18,6 @@ import (
 	"github.com/semi-technologies/weaviate/entities/storagestate"
 )
 
-func (s *Shard) initStatus() {
-	s.statusLock.Lock()
-	defer s.statusLock.Unlock()
-
-	s.status = storagestate.StatusReady
-}
-
 func (s *Shard) getStatus() storagestate.Status {
 	s.statusLock.Lock()
 	defer s.statusLock.Unlock()

adapters/repos/db/shard_test.go

@@ -57,7 +57,7 @@ func TestShard_UpdateStatus(t *testing.T) {
 }
 
 func TestReadOnlyShard_HaltCompaction(t *testing.T) {
-	amount := 100000
+	amount := 10000
 	sizePerValue := 8
 	bucketName := "testbucket"
 
@@ -97,6 +97,8 @@ func TestReadOnlyShard_HaltCompaction(t *testing.T) {
 			assert.Nil(t, err)
 			time.Sleep(time.Microsecond)
 		}
+
+		t.Logf("insertion complete!")
 	})
 
 	t.Run("halt compaction with readonly status", func(t *testing.T) {
@@ -123,12 +125,11 @@ func TestReadOnlyShard_HaltCompaction(t *testing.T) {
 			require.Nil(t, err)
 
 			require.Equal(t, numSegments, len(entries))
+			t.Logf("iteration %d, sleeping", i)
 			time.Sleep(time.Second)
 		}
 	})
 
-	t.Log("shutdown shard")
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-	require.Nil(t, shd.shutdown(ctx))
+	t.Logf("drop shard")
+	require.Nil(t, shd.drop(true))
 }

test/integration/run.sh

@@ -2,11 +2,6 @@
 
 set -e 
 
-echo 'BEGIN TEMP DEBUG ON CI'
-echo 'run stalling test in isolation'
-(cd adapters/repos/db && go test -tags integrationTest -run Halt)
-
-echo 'END TEMP DEBUG ON CI'
 norestart=false
 includeslow=false