Remove reference_cast in make_boxed_from_unboxed_functor (#51319)

Summary:
Pull Request resolved: pytorch/pytorch#51319

We were going out of our way to accommodate `IValue::to<Tensor>` returning a copy of the inner Tensor. `IValue::toTensor` is capable of returning a reference without copying, so if we use it directly, we can allow kernels that want to take `Tensor &` to do so!
As a bonus, we get reduced build times.
ghstack-source-id: 121378961

Reviewed By: bhosmer

Differential Revision: D26138549

fbshipit-source-id: b0f830527da360c542c815bef2f7e1692615b32a

torchaudio/csrc/sox/effects.cpp

@@ -91,9 +91,9 @@ std::tuple<torch::Tensor, int64_t> apply_effects_tensor(
 std::tuple<torch::Tensor, int64_t> apply_effects_file(
     const std::string path,
     std::vector<std::vector<std::string>> effects,
-    c10::optional<bool>& normalize,
-    c10::optional<bool>& channels_first,
-    c10::optional<std::string>& format) {
+    c10::optional<bool> normalize,
+    c10::optional<bool> channels_first,
+    const c10::optional<std::string>& format) {
   // Open input file
   SoxFormat sf(sox_open_read(
       path.c_str(),

torchaudio/csrc/sox/effects.h

@@ -24,9 +24,9 @@ std::tuple<torch::Tensor, int64_t> apply_effects_tensor(
 std::tuple<torch::Tensor, int64_t> apply_effects_file(
     const std::string path,
     std::vector<std::vector<std::string>> effects,
-    c10::optional<bool>& normalize,
-    c10::optional<bool>& channels_first,
-    c10::optional<std::string>& format);
+    c10::optional<bool> normalize,
+    c10::optional<bool> channels_first,
+    const c10::optional<std::string>& format);
 
 #ifdef TORCH_API_INCLUDE_EXTENSION_H
 

torchaudio/csrc/sox/io.cpp

@@ -47,7 +47,7 @@ std::string get_encoding(sox_encoding_t encoding) {
 
 std::tuple<int64_t, int64_t, int64_t, int64_t, std::string> get_info_file(
     const std::string& path,
-    c10::optional<std::string>& format) {
+    const c10::optional<std::string>& format) {
   SoxFormat sf(sox_open_read(
       path.c_str(),
       /*signal=*/nullptr,
@@ -69,8 +69,8 @@ std::tuple<int64_t, int64_t, int64_t, int64_t, std::string> get_info_file(
 namespace {
 
 std::vector<std::vector<std::string>> get_effects(
-    c10::optional<int64_t>& frame_offset,
-    c10::optional<int64_t>& num_frames) {
+    const c10::optional<int64_t>& frame_offset,
+    const c10::optional<int64_t>& num_frames) {
   const auto offset = frame_offset.value_or(0);
   if (offset < 0) {
     throw std::runtime_error(
@@ -101,11 +101,11 @@ std::vector<std::vector<std::string>> get_effects(
 
 std::tuple<torch::Tensor, int64_t> load_audio_file(
     const std::string& path,
-    c10::optional<int64_t>& frame_offset,
-    c10::optional<int64_t>& num_frames,
-    c10::optional<bool>& normalize,
-    c10::optional<bool>& channels_first,
-    c10::optional<std::string>& format) {
+    const c10::optional<int64_t>& frame_offset,
+    const c10::optional<int64_t>& num_frames,
+    c10::optional<bool> normalize,
+    c10::optional<bool> channels_first,
+    const c10::optional<std::string>& format) {
   auto effects = get_effects(frame_offset, num_frames);
   return torchaudio::sox_effects::apply_effects_file(
       path, effects, normalize, channels_first, format);

torchaudio/csrc/sox/io.h

@@ -13,15 +13,15 @@ namespace sox_io {
 
 std::tuple<int64_t, int64_t, int64_t, int64_t, std::string> get_info_file(
     const std::string& path,
-    c10::optional<std::string>& format);
+    const c10::optional<std::string>& format);
 
 std::tuple<torch::Tensor, int64_t> load_audio_file(
     const std::string& path,
-    c10::optional<int64_t>& frame_offset,
-    c10::optional<int64_t>& num_frames,
-    c10::optional<bool>& normalize,
-    c10::optional<bool>& channels_first,
-    c10::optional<std::string>& format);
+    const c10::optional<int64_t>& frame_offset,
+    const c10::optional<int64_t>& num_frames,
+    c10::optional<bool> normalize,
+    c10::optional<bool> channels_first,
+    const c10::optional<std::string>& format);
 
 void save_audio_file(
     const std::string& path,