diff --git a/puppet.go b/puppet.go
index 1ecc2409..627d3af0 100644
--- a/puppet.go
+++ b/puppet.go
@@ -325,8 +325,8 @@ func (puppet *Puppet) updateAvatar(ctx context.Context, source *User, info *type
 			// TODO what to do? 🤔
 			return false
 		}
-		puppet.AvatarSet = false
 		puppet.AvatarPath = ""
+		puppet.AvatarHash = info.ContactAvatar.Hash
 	} else {
 		if puppet.AvatarPath == info.Profile.AvatarPath && puppet.AvatarSet {
 			return false
@@ -354,19 +354,19 @@ func (puppet *Puppet) updateAvatar(ctx context.Context, source *User, info *type
 			return true
 		}
 		avatarContentType = http.DetectContentType(avatarData)
+		puppet.AvatarPath = info.Profile.AvatarPath
+		hash := sha256.Sum256(avatarData)
+		newHash := hex.EncodeToString(hash[:])
+		if puppet.AvatarHash == newHash && puppet.AvatarSet {
+			log.Debug().
+				Str("avatar_hash", newHash).
+				Str("new_avatar_path", puppet.AvatarPath).
+				Msg("Avatar path changed, but hash didn't")
+			// Path changed, but actual avatar didn't
+			return true
+		}
+		puppet.AvatarHash = newHash
 	}
-	hash := sha256.Sum256(avatarData)
-	newHash := hex.EncodeToString(hash[:])
-	if puppet.AvatarHash == newHash && puppet.AvatarSet {
-		log.Debug().
-			Str("avatar_hash", newHash).
-			Str("new_avatar_path", puppet.AvatarPath).
-			Msg("Avatar path changed, but hash didn't")
-		// Path changed, but actual avatar didn't
-		return true
-	}
-	puppet.AvatarPath = info.Profile.AvatarPath
-	puppet.AvatarHash = newHash
 	puppet.AvatarSet = false
 	puppet.AvatarURL = id.ContentURI{}
 	resp, err := puppet.DefaultIntent().UploadBytes(ctx, avatarData, avatarContentType)
@@ -383,7 +383,7 @@ func (puppet *Puppet) updateAvatar(ctx context.Context, source *User, info *type
 		return true
 	}
 	log.Debug().
-		Str("avatar_hash", newHash).
+		Str("avatar_hash", puppet.AvatarHash).
 		Stringer("avatar_mxc", resp.ContentURI).
 		Msg("Avatar updated successfully")
 	puppet.AvatarSet = true