-
Notifications
You must be signed in to change notification settings - Fork 37
/
request.h
138 lines (111 loc) · 4.81 KB
/
request.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
#ifndef SRC_BERGAMOT_REQUEST_H_
#define SRC_BERGAMOT_REQUEST_H_
#include <cassert>
#include <future>
#include <vector>
#include "annotation.h"
#include "cache.h"
#include "common/logging.h"
#include "data/types.h"
#include "definitions.h"
#include "response.h"
#include "response_builder.h"
#include "translator/beam_search.h"
namespace marian {
namespace bergamot {
class TranslationModel;
/// A Request is an internal representation used to represent a request after
/// processed by TextProcessor into sentences constituted by marian::Words.
///
/// The batching mechanism (BatchingPool) draws from multiple Requests and compiles
/// sentences into a batch. When a batch completes translation (at
/// BatchTranslator, intended in a different thread), backward propogation
/// happens through:
///
/// ```cpp
/// Batch::completeBatch(...)
/// -> RequestSentence::completeSentence(..)
/// -> Request::processHistory(...)
/// ```
///
/// When all sentences in a Request are completed, responseBuilder is
/// triggered with the compiled Histories, to construct the Response
/// corresponding to the Request and set value of the promise which triggers the
/// future at client.
class Request {
public:
/// Constructs an internal representation of the Request identified by Id,
/// processed Segments and accepts a callback (ResponseBuilder) which builds
/// the Response upon completion of the Request.
///
///
/// @param [in] Id: Identifier assigned to Request by Service.
/// @param [in] model: TranslationModel for identifying a unique translation unit key (model, words in a sentence) for
/// cache.
/// @param [in] segments: Each segment is a unit to be translated.
/// @param [in] responseBuilder: Callback function (of ResponseBuilder type)
/// to be triggered upon the completion of translation of all units in a
/// Request.
/// @param [in] cache: Cache supplied externally to attempt to fetch translations or store them after completion for
/// reuse later.
Request(size_t Id, const TranslationModel &model, Segments &&segments, ResponseBuilder &&responseBuilder,
std::optional<TranslationCache> &cache);
/// Obtain the count of tokens in the segment correponding to index. Used to
/// insert sentence from multiple requests into the corresponding size bucket.
size_t segmentTokens(size_t index) const;
/// Obtain number of segments in a request.
size_t numSegments() const;
/// Obtains segment corresponding to index to create a batch of segments
/// among several requests.
Segment getSegment(size_t index) const;
/// For notions of priority among requests, used to enable std::set in
/// BatchingPool.
bool operator<(const Request &request) const;
/// Processes a history obtained after translating in a heterogenous batch
/// compiled from requests.
void processHistory(size_t index, Ptr<History> history);
bool cacheHitPrefilled(size_t index) const { return histories_[index] != nullptr; }
private:
size_t Id_;
/// TranslationModel associated with this request
const TranslationModel &model_;
/// Multiple translation-workers can concurrently access the same Request. The
/// following atomic atomically operates on the variable holding sentences
/// remaining to be translated.
std::atomic<int> counter_;
/// segments_ hold the sentences processed into Words which generated from
/// input string.
Segments segments_;
/// histories_ is a buffer which eventually stores the translations of each
/// segment in the corresponding index.
std::vector<Ptr<History>> histories_;
/// Constructing Response requires the vocabs_ used to generate Request.
/// std::vector<Ptr<Vocab const>> *vocabs_;
ResponseBuilder responseBuilder_;
/// Cache used to hold unit translations. If nullopt, means no-caching.
std::optional<TranslationCache> &cache_;
};
/// A RequestSentence provides a view to a sentence within a Request. Existence
/// of this class allows the sentences and associated information to be kept
/// within Request, while batching mechanism (BatchingPool) compiles Batch from
/// RequestSentence-s coming from different Requests.
class RequestSentence {
public:
RequestSentence(size_t, Ptr<Request>);
/// Number of tokens in the segment this RequestSentence represents. Used to
/// order by length in batching.
size_t numTokens() const;
/// Accessor to the segment represented by the RequestSentence.
Segment getUnderlyingSegment() const;
/// Forwards history to Request to set history corresponding to this
/// RequestSentence.
void completeSentence(Ptr<History> history);
friend bool operator<(const RequestSentence &a, const RequestSentence &b);
private:
size_t index_;
Ptr<Request> request_;
};
typedef std::vector<RequestSentence> RequestSentences;
} // namespace bergamot
} // namespace marian
#endif // SRC_BERGAMOT_REQUEST_H_