diff --git a/pkg/image/controller/signature/container_image_downloader.go b/pkg/image/controller/signature/container_image_downloader.go
index b657dbe57..843c65226 100644
--- a/pkg/image/controller/signature/container_image_downloader.go
+++ b/pkg/image/controller/signature/container_image_downloader.go
@@ -36,7 +36,11 @@ func (s *containerImageSignatureDownloader) DownloadImageSignatures(image *image
 	if err != nil {
 		return nil, err
 	}
-	source, err := reference.NewImageSource(nil, nil)
+
+	ctx, cancel := context.WithTimeout(s.ctx, s.timeout)
+	defer cancel()
+
+	source, err := reference.NewImageSource(ctx, nil)
 	if err != nil {
 		// In case we fail to talk to registry to get the image metadata (private
 		// registry, internal registry, etc...), do not fail with error to avoid
@@ -46,9 +50,6 @@ func (s *containerImageSignatureDownloader) DownloadImageSignatures(image *image
 	}
 	defer source.Close()
 
-	ctx, cancel := context.WithTimeout(s.ctx, s.timeout)
-	defer cancel()
-
 	signatures, err := source.GetSignatures(ctx, nil)
 	if err != nil {
 		klog.V(4).Infof("Failed to get signatures for %v due to: %v", source.Reference(), err)