-
Notifications
You must be signed in to change notification settings - Fork 3.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
ARROW-1408: [C++] IPC public API cleanup, refactoring. Add SerializeS…
…chema, ReadSchema public APIs This is mostly moving code around. In reviewing I recommend focusing on the public headers. There were a number of places where it is more consistent to use naked pointers versus shared_ptr. Also some constructors were returning shared_ptr to subclass, where it would be simpler for clients to return a pointer to base. This includes ARROW-1376 and ARROW-1406 Author: Wes McKinney <wes.mckinney@twosigma.com> Closes #988 from wesm/ARROW-1408 and squashes the following commits: b156767 [Wes McKinney] Fix up glib bindings, undeprecate some APIs 4bdebfa [Wes McKinney] Add serialize methods to RecordBatch, Schema. Test round trip ef12e0f [Wes McKinney] Fix a valgrind warning 73d30c9 [Wes McKinney] Better comments 8597b96 [Wes McKinney] Remove API that was never intended to be public, unlikely to be used anywhere 122a759 [Wes McKinney] Refactoring sweep and cleanup of public IPC API. Move non-public APIs from metadata.h to metadata-internal.h and create message.h, dictionary.h b646f96 [Wes McKinney] Set device in more places
- Loading branch information
Showing
39 changed files
with
1,191 additions
and
678 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,85 @@ | ||
| // Licensed to the Apache Software Foundation (ASF) under one | ||
| // or more contributor license agreements. See the NOTICE file | ||
| // distributed with this work for additional information | ||
| // regarding copyright ownership. The ASF licenses this file | ||
| // to you under the Apache License, Version 2.0 (the | ||
| // "License"); you may not use this file except in compliance | ||
| // with the License. You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, | ||
| // software distributed under the License is distributed on an | ||
| // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| // KIND, either express or implied. See the License for the | ||
| // specific language governing permissions and limitations | ||
| // under the License. | ||
|
|
||
| #include "arrow/ipc/dictionary.h" | ||
|
|
||
| #include <cstdint> | ||
| #include <memory> | ||
| #include <sstream> | ||
|
|
||
| #include "arrow/array.h" | ||
| #include "arrow/status.h" | ||
| #include "arrow/type.h" | ||
|
|
||
| namespace arrow { | ||
| namespace ipc { | ||
|
|
||
| DictionaryMemo::DictionaryMemo() {} | ||
|
|
||
| // Returns KeyError if dictionary not found | ||
| Status DictionaryMemo::GetDictionary(int64_t id, | ||
| std::shared_ptr<Array>* dictionary) const { | ||
| auto it = id_to_dictionary_.find(id); | ||
| if (it == id_to_dictionary_.end()) { | ||
| std::stringstream ss; | ||
| ss << "Dictionary with id " << id << " not found"; | ||
| return Status::KeyError(ss.str()); | ||
| } | ||
| *dictionary = it->second; | ||
| return Status::OK(); | ||
| } | ||
|
|
||
| int64_t DictionaryMemo::GetId(const std::shared_ptr<Array>& dictionary) { | ||
| intptr_t address = reinterpret_cast<intptr_t>(dictionary.get()); | ||
| auto it = dictionary_to_id_.find(address); | ||
| if (it != dictionary_to_id_.end()) { | ||
| // Dictionary already observed, return the id | ||
| return it->second; | ||
| } else { | ||
| int64_t new_id = static_cast<int64_t>(dictionary_to_id_.size()); | ||
| dictionary_to_id_[address] = new_id; | ||
| id_to_dictionary_[new_id] = dictionary; | ||
| return new_id; | ||
| } | ||
| } | ||
|
|
||
| bool DictionaryMemo::HasDictionary(const std::shared_ptr<Array>& dictionary) const { | ||
| intptr_t address = reinterpret_cast<intptr_t>(dictionary.get()); | ||
| auto it = dictionary_to_id_.find(address); | ||
| return it != dictionary_to_id_.end(); | ||
| } | ||
|
|
||
| bool DictionaryMemo::HasDictionaryId(int64_t id) const { | ||
| auto it = id_to_dictionary_.find(id); | ||
| return it != id_to_dictionary_.end(); | ||
| } | ||
|
|
||
| Status DictionaryMemo::AddDictionary(int64_t id, | ||
| const std::shared_ptr<Array>& dictionary) { | ||
| if (HasDictionaryId(id)) { | ||
| std::stringstream ss; | ||
| ss << "Dictionary with id " << id << " already exists"; | ||
| return Status::KeyError(ss.str()); | ||
| } | ||
| intptr_t address = reinterpret_cast<intptr_t>(dictionary.get()); | ||
| id_to_dictionary_[id] = dictionary; | ||
| dictionary_to_id_[address] = id; | ||
| return Status::OK(); | ||
| } | ||
|
|
||
| } // namespace ipc | ||
| } // namespace arrow |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| // Licensed to the Apache Software Foundation (ASF) under one | ||
| // or more contributor license agreements. See the NOTICE file | ||
| // distributed with this work for additional information | ||
| // regarding copyright ownership. The ASF licenses this file | ||
| // to you under the Apache License, Version 2.0 (the | ||
| // "License"); you may not use this file except in compliance | ||
| // with the License. You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, | ||
| // software distributed under the License is distributed on an | ||
| // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| // KIND, either express or implied. See the License for the | ||
| // specific language governing permissions and limitations | ||
| // under the License. | ||
|
|
||
| // Tools for dictionaries in IPC context | ||
|
|
||
| #ifndef ARROW_IPC_DICTIONARY_H | ||
| #define ARROW_IPC_DICTIONARY_H | ||
|
|
||
| #include <cstdint> | ||
| #include <memory> | ||
| #include <string> | ||
| #include <unordered_map> | ||
| #include <vector> | ||
|
|
||
| #include "arrow/status.h" | ||
| #include "arrow/util/macros.h" | ||
| #include "arrow/util/visibility.h" | ||
|
|
||
| namespace arrow { | ||
|
|
||
| class Array; | ||
| class Buffer; | ||
| class Field; | ||
|
|
||
| namespace io { | ||
|
|
||
| class InputStream; | ||
| class OutputStream; | ||
| class RandomAccessFile; | ||
|
|
||
| } // namespace io | ||
|
|
||
| namespace ipc { | ||
|
|
||
| using DictionaryMap = std::unordered_map<int64_t, std::shared_ptr<Array>>; | ||
| using DictionaryTypeMap = std::unordered_map<int64_t, std::shared_ptr<Field>>; | ||
|
|
||
| // Memoization data structure for handling shared dictionaries | ||
| class ARROW_EXPORT DictionaryMemo { | ||
| public: | ||
| DictionaryMemo(); | ||
|
|
||
| // Returns KeyError if dictionary not found | ||
| Status GetDictionary(int64_t id, std::shared_ptr<Array>* dictionary) const; | ||
|
|
||
| /// Return id for dictionary, computing new id if necessary | ||
| int64_t GetId(const std::shared_ptr<Array>& dictionary); | ||
|
|
||
| bool HasDictionary(const std::shared_ptr<Array>& dictionary) const; | ||
| bool HasDictionaryId(int64_t id) const; | ||
|
|
||
| // Add a dictionary to the memo with a particular id. Returns KeyError if | ||
| // that dictionary already exists | ||
| Status AddDictionary(int64_t id, const std::shared_ptr<Array>& dictionary); | ||
|
|
||
| const DictionaryMap& id_to_dictionary() const { return id_to_dictionary_; } | ||
|
|
||
| int size() const { return static_cast<int>(id_to_dictionary_.size()); } | ||
|
|
||
| private: | ||
| // Dictionary memory addresses, to track whether a dictionary has been seen | ||
| // before | ||
| std::unordered_map<intptr_t, int64_t> dictionary_to_id_; | ||
|
|
||
| // Map of dictionary id to dictionary array | ||
| DictionaryMap id_to_dictionary_; | ||
|
|
||
| DISALLOW_COPY_AND_ASSIGN(DictionaryMemo); | ||
| }; | ||
|
|
||
| } // namespace ipc | ||
| } // namespace arrow | ||
|
|
||
| #endif // ARROW_IPC_DICTIONARY_H |
Oops, something went wrong.