diff --git a/cpp/src/arrow/row_conversion.h b/cpp/src/arrow/row_conversion.h
index a7c966ad572a6..665a9f9530a72 100644
--- a/cpp/src/arrow/row_conversion.h
+++ b/cpp/src/arrow/row_conversion.h
@@ -31,7 +31,7 @@ namespace arrow {
 /// The derived conversion functions take a batch_size parameter that determines
 /// how many rows convert at once. This allows you to do the conversion in smaller
 /// batches.
-template <typename RowType, typename ContainerType = std::vector<RowType>>
+template <typename RowType, template <typename...> class ContainerType = std::vector>
 class ToRowConverter {
  public:
   /// \brief Convert a record batch into a vector of rows.
@@ -40,17 +40,16 @@ class ToRowConverter {
   ///
   /// \param batch Record batch to convert
   /// \return A vector of rows containing all data in batch
-  virtual Result<ContainerType> ConvertToVector(std::shared_ptr<RecordBatch> batch) {
-    return arrow::Status::NotImplemented("Conversion not implemented");
-  }
+  virtual Result<ContainerType<RowType>> ConvertToVector(
+      std::shared_ptr<RecordBatch> batch) = 0;
 
   /// \brief Convert a record batch into a vector of rows.
   ///
   /// \param batch Record batch to convert
   /// \param batch_size Number of rows to convert at once
   /// \return A vector of rows containing all data in batch
-  Result<ContainerType> ConvertToVector(std::shared_ptr<RecordBatch> batch,
-                                        size_t batch_size) {
+  Result<ContainerType<RowType>> ConvertToVector(std::shared_ptr<RecordBatch> batch,
+                                                 size_t batch_size) {
     ARROW_ASSIGN_OR_RAISE(std::shared_ptr<Table> table,
                           Table::FromRecordBatches({batch}));
     return ConvertToVector(table, batch_size);
@@ -61,8 +60,9 @@ class ToRowConverter {
   /// \param table Table to convert
   /// \param batch_size Number of rows to convert at once
   /// \return A vector of rows containing all data in table
-  Result<ContainerType> ConvertToVector(std::shared_ptr<Table> table, size_t batch_size) {
-    ContainerType out;
+  Result<ContainerType<RowType>> ConvertToVector(std::shared_ptr<Table> table,
+                                                 size_t batch_size) {
+    ContainerType<RowType> out;
 
     TableBatchReader batch_reader(*table);
     batch_reader.set_chunksize(batch_size);
@@ -122,7 +122,7 @@ class ToRowConverter {
 ///
 /// \tparam RowType type that contains a single row
 /// \tparam ContainerType optional alternative vector implementation
-template <typename RowType, typename ContainerType = std::vector<RowType>>
+template <typename RowType, template <typename...> class ContainerType = std::vector>
 class FromRowConverter {
  public:
   /// \brief Convert a vector of rows to a record batch.
@@ -133,16 +133,14 @@ class FromRowConverter {
   /// \param schema Schema of record batch to convert to
   /// \return Arrow result with record batch containing all rows
   virtual Result<std::shared_ptr<RecordBatch>> ConvertToRecordBatch(
-      const ContainerType& rows, std::shared_ptr<Schema> schema) {
-    return arrow::Status::NotImplemented("Conversion not implemented");
-  }
+      const ContainerType<RowType>& rows, std::shared_ptr<Schema> schema) = 0;
 
   /// \brief Convert a vector of rows to an Arrow table
   ///
   /// \param rows Vector of rows to convert
   /// \param schema Schema of record batch to convert to
   /// \return Arrow result with table containing all rows
-  Result<std::shared_ptr<Table>> ConvertToTable(const ContainerType& rows,
+  Result<std::shared_ptr<Table>> ConvertToTable(const ContainerType<RowType>& rows,
                                                 std::shared_ptr<Schema> schema) {
     ARROW_ASSIGN_OR_RAISE(std::shared_ptr<RecordBatch> batch,
                           ConvertToRecordBatch(rows, schema));
@@ -159,7 +157,7 @@ class FromRowConverter {
       Iterator<RowType> row_iter, std::shared_ptr<Schema> schema, size_t batch_size) {
     auto get_next_batch = [row_iter, batch_size, schema,
                            this]() -> Result<std::shared_ptr<RecordBatch>> {
-      ContainerType rows;
+      ContainerType<RowType> rows;
       RowType row;
       rows.reserve(batch_size);