Skip to content
This repository was archived by the owner on Apr 10, 2025. It is now read-only.

Commit 29d303a

Browse files
crowellcrowell
committed
Add class for encoding/decoding brotli
not hooked up to anything yet.
1 parent 94a3ce1 commit 29d303a

File tree

4 files changed

+359
-1
lines changed

4 files changed

+359
-1
lines changed

‎DEPS‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,10 @@ vars = {
9494

9595
"libpng_src": "https://github.com/glennrp/libpng.git",
9696
"libpng_revision": "@2d2fe5e9efbaba6339742807b93a2646aae88475",
97+
98+
# Brotli v0.2.0
99+
"brotli_src": "https://github.com/google/brotli.git",
100+
"brotli_revision": "@7f7a2fb48cec63c0459ec6b6e7260810bfb01819",
97101
}
98102

99103
deps = {
@@ -233,7 +237,9 @@ deps = {
233237
Var("domain_registry_provider_src") +
234238
Var("domain_registry_provider_revision"),
235239

236-
"src/third_party/libpng/src": Var("libpng_src") + Var("libpng_revision")
240+
"src/third_party/libpng/src": Var("libpng_src") + Var("libpng_revision"),
241+
242+
"src/third_party/brotli/src": Var("brotli_src") + Var("brotli_revision")
237243
}
238244

239245

‎pagespeed/kernel/util/brotli_inflater.cc‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
/*
2+
* Copyright 2016 Google Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
// Author: jcrowell@google.com (Jeffrey Crowell)
18+
19+
#include "pagespeed/kernel/util/brotli_inflater.h"
20+
21+
#include <cstddef>
22+
#include <cstdlib>
23+
#include "base/logging.h"
24+
#include "third_party/brotli/src/dec/decode.h"
25+
#include "third_party/brotli/src/enc/encode.h"
26+
#include "pagespeed/kernel/base/message_handler.h"
27+
#include "pagespeed/kernel/base/string.h"
28+
#include "pagespeed/kernel/base/string_writer.h"
29+
#include "pagespeed/kernel/base/stack_buffer.h"
30+
#include "pagespeed/kernel/base/writer.h"
31+
32+
namespace net_instaweb {
33+
34+
BrotliInflater::BrotliInflater()
35+
: state_used_(false), brotli_state_(new BrotliState) {
36+
BrotliStateInit(brotli_state_.get());
37+
}
38+
39+
BrotliInflater::~BrotliInflater() {
40+
BrotliStateCleanup(brotli_state_.get());
41+
}
42+
43+
void BrotliInflater::ResetState() {
44+
if (state_used_) {
45+
BrotliStateCleanup(brotli_state_.get());
46+
BrotliStateInit(brotli_state_.get());
47+
}
48+
state_used_ = true;
49+
}
50+
51+
bool BrotliInflater::Compress(StringPiece in, int compression_level,
52+
MessageHandler* handler, Writer* writer) {
53+
// For creating a BrotliStringOut.
54+
GoogleString out_str;
55+
// Set the compression level ("quality" in brotli terms).
56+
util::compression::brotli::BrotliParams params;
57+
params.quality = compression_level;
58+
util::compression::brotli::BrotliMemIn brotli_input(in.data(), in.length());
59+
util::compression::brotli::BrotliStringOut brotli_output(
60+
&out_str, std::numeric_limits<int>::max());
61+
// Compress in one shot with BrotliCompress.
62+
if (util::compression::brotli::BrotliCompress(params, &brotli_input,
63+
&brotli_output)) {
64+
return writer->Write(out_str, handler);
65+
}
66+
return false;
67+
}
68+
69+
bool BrotliInflater::Compress(StringPiece in, MessageHandler* handler,
70+
Writer* writer) {
71+
// Default quality is 11 for brotli.
72+
return BrotliInflater::Compress(in, 11, handler, writer);
73+
}
74+
75+
bool BrotliInflater::DecompressHelper(StringPiece in, MessageHandler* handler,
76+
Writer* writer) {
77+
// Mostly taken from BrotliDecompress in the tool "bro".
78+
// https://raw.githubusercontent.com/google/brotli/v0.2.0/tools/bro.cc
79+
char output[kStackBufferSize];
80+
size_t total_out = 0;
81+
size_t available_in = in.length();
82+
const char* next_in = in.data();
83+
BrotliResult result = BROTLI_RESULT_NEEDS_MORE_INPUT;
84+
ResetState();
85+
while (result != BROTLI_RESULT_SUCCESS) {
86+
size_t available_out = sizeof(output);
87+
char* next_out = output;
88+
result = BrotliDecompressStream(
89+
&available_in, reinterpret_cast<const unsigned char**>(&next_in),
90+
&available_out, reinterpret_cast<unsigned char**>(&next_out),
91+
&total_out, brotli_state_.get());
92+
CHECK(next_in >= in.data());
93+
CHECK_LE(available_out, sizeof(output));
94+
in.remove_prefix(next_in - in.data());
95+
switch (result) {
96+
case BROTLI_RESULT_NEEDS_MORE_INPUT:
97+
// We should never hit this case because the compressed input isn't
98+
// streamed.
99+
handler->Message(kWarning, "BROTLI_RESULT_NEEDS_MORE_INPUT");
100+
return false;
101+
case BROTLI_RESULT_ERROR:
102+
// Decompressing failed.
103+
handler->Message(kError, "BROTLI_RESULT_ERROR");
104+
return false;
105+
case BROTLI_RESULT_NEEDS_MORE_OUTPUT:
106+
// Need to flush the output buffer to the writer.
107+
break;
108+
case BROTLI_RESULT_SUCCESS:
109+
// Decompression succeeded, write out the last chunk if needed.
110+
break;
111+
}
112+
StringPiece chunk(output, sizeof(output) - available_out);
113+
if (!chunk.empty() && !writer->Write(chunk, handler)) {
114+
return false;
115+
}
116+
}
117+
return true; // BROTLI_RESULT_SUCCESS
118+
}
119+
120+
bool BrotliInflater::Decompress(StringPiece in, MessageHandler* handler,
121+
Writer* writer) {
122+
BrotliInflater brotli;
123+
return brotli.DecompressHelper(in, handler, writer);
124+
}
125+
126+
} // namespace net_instaweb

‎pagespeed/kernel/util/brotli_inflater.h‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/*
2+
* Copyright 2016 Google Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
// Author: jcrowell@google.com (Jeffrey Crowell)
18+
19+
#ifndef PAGESPEED_KERNEL_UTIL_BROTLI_INFLATER_H_
20+
#define PAGESPEED_KERNEL_UTIL_BROTLI_INFLATER_H_
21+
22+
#include <cstddef>
23+
#include "third_party/brotli/src/dec/state.h"
24+
#include "pagespeed/kernel/base/basictypes.h"
25+
#include "pagespeed/kernel/base/scoped_ptr.h"
26+
#include "pagespeed/kernel/base/string_util.h"
27+
28+
namespace net_instaweb {
29+
30+
class Writer;
31+
class MessageHandler;
32+
33+
// TODO(jcrowell): Add compression interface that can handle multiple
34+
// compressors (gzip/deflate, brotli, etc.).
35+
class BrotliInflater {
36+
public:
37+
BrotliInflater();
38+
~BrotliInflater();
39+
40+
// Compresses a StringPiece, writing output to Writer. Returns false
41+
// if there was some kind of failure, though none are expected.
42+
// If no compression level is specified, the default of 11 (maximum
43+
// compression/highest quality) is used.
44+
// TODO(jcrowell): add api that takes in &out_string as an argument, to remove
45+
// string copy.
46+
static bool Compress(StringPiece in, MessageHandler* handler, Writer* writer);
47+
static bool Compress(StringPiece in, int compression_level,
48+
MessageHandler* handler, Writer* writer);
49+
50+
// Decompresses a StringPiece, writing output to Writer. Returns false
51+
// if there was some kind of failure, such as a corrupt input.
52+
static bool Decompress(StringPiece in, MessageHandler* handler,
53+
Writer* writer);
54+
bool DecompressHelper(StringPiece in, MessageHandler* handler,
55+
Writer* writer);
56+
57+
// TODO(jcrowell): Add API with properly sized output buffer (taken from
58+
// X-Original-Content-Length).
59+
60+
private:
61+
// Reset the BrotliState.
62+
void ResetState();
63+
64+
// Keep track of if the internal state is "dirty", if so, refreshed by
65+
// ResetState() before Decompression.
66+
bool state_used_;
67+
scoped_ptr<BrotliState> brotli_state_;
68+
69+
DISALLOW_COPY_AND_ASSIGN(BrotliInflater);
70+
};
71+
72+
} // namespace net_instaweb
73+
74+
#endif // PAGESPEED_KERNEL_UTIL_BROTLI_INFLATER_H_

‎pagespeed/kernel/util/brotli_inflater_test.cc‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
/*
2+
* Copyright 2016 Google Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
// Author: jcrowell@google.com (Jeffrey Crowell)
18+
19+
#include "pagespeed/kernel/util/brotli_inflater.h"
20+
21+
#include <cstddef>
22+
#include "pagespeed/kernel/base/gtest.h"
23+
#include "pagespeed/kernel/base/message_handler_test_base.h"
24+
#include "pagespeed/kernel/base/null_mutex.h"
25+
#include "pagespeed/kernel/util/simple_random.h"
26+
#include "pagespeed/kernel/base/stack_buffer.h"
27+
#include "pagespeed/kernel/base/string.h"
28+
#include "pagespeed/kernel/base/string_util.h"
29+
#include "pagespeed/kernel/base/string_writer.h"
30+
31+
namespace net_instaweb {
32+
33+
namespace {
34+
35+
// Generated with command line tool "bro".
36+
static const char kHello[] = "hello\n";
37+
static const char kHelloBrotli[] = "\x8b\x02\x80\x68\x65\x6c\x6c\x6f\x0a\x03";
38+
// Use the highest quality by default (11).
39+
static const int kCompressionLevel = 11;
40+
41+
// Writer that returns false on Write().
42+
class StringWriterReturningFalse : public StringWriter {
43+
public:
44+
explicit StringWriterReturningFalse(GoogleString* str) : StringWriter(str) {}
45+
bool Write(const StringPiece& str,
46+
net_instaweb::MessageHandler* message_handler) {
47+
StringWriter::Write(str, message_handler);
48+
return false;
49+
}
50+
};
51+
52+
TEST(BrotliInflater, TestBrotliDecompress) {
53+
// Compress and decompress a simple string.
54+
GoogleString decompressed;
55+
TestMessageHandler handler;
56+
StringWriter decompress_writer(&decompressed);
57+
StringPiece compressed(kHelloBrotli, sizeof(kHelloBrotli));
58+
EXPECT_TRUE(
59+
BrotliInflater::Decompress(compressed, &handler, &decompress_writer));
60+
EXPECT_STREQ(kHello, decompressed);
61+
EXPECT_EQ(0, handler.messages().size());
62+
}
63+
64+
TEST(BrotliInflater, TestFailedWriteBrotliDecompress) {
65+
// Use a writer that returns failure on write.
66+
GoogleString decompressed;
67+
TestMessageHandler handler;
68+
StringWriterReturningFalse decompress_writer(&decompressed);
69+
StringPiece compressed(kHelloBrotli, sizeof(kHelloBrotli));
70+
EXPECT_FALSE(
71+
BrotliInflater::Decompress(compressed, &handler, &decompress_writer));
72+
EXPECT_EQ(0, handler.messages().size());
73+
}
74+
75+
TEST(BrotliInflater, TestCorruptInputBrotliDecompress) {
76+
// Take "hello\n" but replace the first 2 bytes with "AB", so it will not be
77+
// valid brotli.
78+
const char kHelloBrotliCorrupt[] = "AB\x80\x68\x65\x6c\x6c\x6f\x0a\x03";
79+
GoogleString decompressed;
80+
TestMessageHandler handler;
81+
StringWriterReturningFalse decompress_writer(&decompressed);
82+
StringPiece compressed(kHelloBrotliCorrupt, sizeof(kHelloBrotliCorrupt));
83+
EXPECT_FALSE(
84+
BrotliInflater::Decompress(compressed, &handler, &decompress_writer));
85+
EXPECT_STREQ("Error: BROTLI_RESULT_ERROR", handler.messages()[0]);
86+
}
87+
88+
TEST(BrotliInflater, TestTruncatedInputBrotliDecompress) {
89+
// Take "hello\n" but truncate the string, so it will not be valid brotli.
90+
const char kHelloBrotliCorrupt[] = "\x8b\x02\x80\x68\x65\x6c\x6c\x6f";
91+
GoogleString decompressed;
92+
TestMessageHandler handler;
93+
StringWriterReturningFalse decompress_writer(&decompressed);
94+
StringPiece compressed(kHelloBrotliCorrupt, sizeof(kHelloBrotliCorrupt));
95+
EXPECT_FALSE(
96+
BrotliInflater::Decompress(compressed, &handler, &decompress_writer));
97+
EXPECT_STREQ("Warning: BROTLI_RESULT_NEEDS_MORE_INPUT",
98+
handler.messages()[0]);
99+
}
100+
101+
TEST(BrotliInflater, TestCompressDecompressSmallString) {
102+
// Compress and decompress a simple string, ensure that it remains unchanged.
103+
StringPiece payload(kHello);
104+
TestMessageHandler handler;
105+
GoogleString compressed, decompressed;
106+
StringWriter compressed_writer(&compressed);
107+
EXPECT_TRUE(BrotliInflater::Compress(payload, kCompressionLevel, &handler,
108+
&compressed_writer));
109+
StringWriter decompressed_writer(&decompressed);
110+
EXPECT_TRUE(
111+
BrotliInflater::Decompress(compressed, &handler, &decompressed_writer));
112+
EXPECT_STREQ(payload, decompressed);
113+
EXPECT_EQ(0, handler.messages().size());
114+
}
115+
116+
TEST(BrotliInflater, TestCompressDecompressLargeString) {
117+
// Compress and decompress a long string of repeated characters that will
118+
// exceed the buffer size.
119+
TestMessageHandler handler;
120+
GoogleString value(5 * kStackBufferSize, 'A');
121+
StringPiece payload(value);
122+
GoogleString compressed, decompressed;
123+
StringWriter compressed_writer(&compressed);
124+
EXPECT_TRUE(BrotliInflater::Compress(payload, kCompressionLevel, &handler,
125+
&compressed_writer));
126+
StringWriter decompressed_writer(&decompressed);
127+
EXPECT_TRUE(
128+
BrotliInflater::Decompress(compressed, &handler, &decompressed_writer));
129+
EXPECT_STREQ(payload, decompressed);
130+
EXPECT_EQ(0, handler.messages().size());
131+
}
132+
133+
TEST(BrotliInflater, TestCompressDecompressLargeStringWithPoorCompression) {
134+
// Compress and decompress a long string of random characters.
135+
TestMessageHandler handler;
136+
SimpleRandom random(new NullMutex);
137+
GoogleString value = random.GenerateHighEntropyString(5 * kStackBufferSize);
138+
StringPiece payload(value);
139+
GoogleString compressed, decompressed;
140+
StringWriter compressed_writer(&compressed);
141+
EXPECT_TRUE(BrotliInflater::Compress(payload, kCompressionLevel, &handler,
142+
&compressed_writer));
143+
StringWriter decompressed_writer(&decompressed);
144+
EXPECT_TRUE(
145+
BrotliInflater::Decompress(compressed, &handler, &decompressed_writer));
146+
EXPECT_STREQ(payload, decompressed);
147+
EXPECT_EQ(0, handler.messages().size());
148+
}
149+
150+
} // namespace
151+
152+
} // namespace net_instaweb

0 commit comments

Comments
 (0)