Skip to content
This repository has been archived by the owner on Apr 21, 2023. It is now read-only.

Commit

Permalink
Add support for the Save-Data client hint.
Browse files Browse the repository at this point in the history 
Fixes #1258

Squash-merge of huibao's work in 1fef4af, e686449, 5da54d8, 6c0be2e,
b98bf9a, 79eb6fa, c0b22d1, and b92ddc1.

Changes nullptr to NULL to support older compilers.
  • Loading branch information
@huibaolin @jeffkaufman
huibaolin authored and jeffkaufman committed Mar 9, 2016
1 parent 5e200ab commit 4d2000c
Show file tree
Hide file tree
Showing 28 changed files with 1,864 additions and 284 deletions.
33 changes: 33 additions & 0 deletions install/debug.conf.template
View file
Expand Up @@ -1685,7 +1685,40 @@ ModPagespeedMessagesDomains Allow localhost
ModPagespeedEnableFilters rewrite_images,rewrite_css
ModPagespeedEnableFilters convert_to_webp_lossless
ModPagespeedEnableFilters in_place_optimize_for_browser
ModPagespeedJpegRecompressionQuality 75
ModPagespeedWebpRecompressionQuality 70
ModPagespeedInPlaceResourceOptimization on
ModPagespeedAllowVaryOn "Accept"
DocumentRoot "@@APACHE_DOC_ROOT@@/mod_pagespeed_example"
ModPagespeedFileCachePath "@@MOD_PAGESPEED_CACHE@@_ipro_for_browser"
</VirtualHost>

<VirtualHost localhost:@@APACHE_SECONDARY_PORT@@>
ServerName ipro-for-browser-vary-on-auto.example.com
ModPagespeedEnableFilters rewrite_images,rewrite_css
ModPagespeedEnableFilters convert_to_webp_lossless
ModPagespeedEnableFilters convert_to_webp_animated
ModPagespeedEnableFilters in_place_optimize_for_browser
ModPagespeedInPlaceResourceOptimization on
# ModPagespeedAllowVaryOn "Auto" # Default is "Auto"
ModPagespeedImageRecompressionQuality 90
ModPagespeedJpegRecompressionQuality 75
ModPagespeedJpegRecompressionQualityForSmallScreens 55
ModPagespeedJpegQualityForSaveData 35
ModPagespeedWebpRecompressionQuality 70
ModPagespeedWebpRecompressionQualityForSmallScreens 50
ModPagespeedWebpQualityForSaveData 30
ModPagespeedWebpAnimatedRecompressionQuality 60
DocumentRoot "@@APACHE_DOC_ROOT@@/mod_pagespeed_example"
ModPagespeedFileCachePath "@@MOD_PAGESPEED_CACHE@@_ipro_for_browser"
</VirtualHost>

<VirtualHost localhost:@@APACHE_SECONDARY_PORT@@>
ServerName ipro-for-browser-vary-on-none.example.com
ModPagespeedEnableFilters rewrite_images,in_place_optimize_for_browser
ModPagespeedInPlaceResourceOptimization on
ModPagespeedAllowVaryOn "None"
ModPagespeedImageRecompressionQuality 75
DocumentRoot "@@APACHE_DOC_ROOT@@/mod_pagespeed_example"
ModPagespeedFileCachePath "@@MOD_PAGESPEED_CACHE@@_ipro_for_browser"
</VirtualHost>
Expand Down
43 changes: 22 additions & 21 deletions net/instaweb/rewriter/cached_result.proto
View file
Expand Up @@ -83,7 +83,7 @@ message InputInfo {
optional string input_content_hash = 7;

// If true, no later filters should process the same input.
optional bool disable_further_processing = 8 [ default = false ];
optional bool disable_further_processing = 8;

// To support invalidation by exact URL, we keep the URL in the metadata.
// This bloats the size a bit, but enables fast invalidation. Compressing
Expand All @@ -95,7 +95,7 @@ message InputInfo {
// is a sequence of input URLs and a filter id. The input array
// tells us which inputs are used to construct this output; it must be
// interpreted using the URL-sequence that was used to form the key.
// Next free tag: 25
// Next free tag: 26
message CachedResult {
// Tags 1-7 are for internal use by output_resource.

Expand All @@ -115,7 +115,7 @@ message CachedResult {
// cache are lost, and it's extremely easy to get it wrong. To catch
// mistakes like that, we mark a CachedResult as 'frozen' upon save,
// and DCHECK any modifications.
optional bool frozen = 5 [ default = false ];
optional bool frozen = 5;

// If the URL is optimizable, this will contain the hash of the encoded
// resource.
Expand Down Expand Up @@ -176,7 +176,7 @@ message CachedResult {
// When this is true, the filter should canonicalize the url to the contents
// of url and skip the default resource-based rendering (eg when we recognize
// a JS library and canonicalize its URL).
optional bool canonicalize_url = 19 [ default = false ];
optional bool canonicalize_url = 19;

// Stores the size of the rewritten resource.
optional int64 size = 20;
Expand All @@ -185,26 +185,20 @@ message CachedResult {
// HTML as comments if the debug filter is enabled.
repeated string debug_message = 21;

// Indicate what level of webp support the given image would require if
// arbitrary webp conversion were enabled. Note that if we attempt webp
// conversion unsuccessfully, this *could* (but need not) be WEBP_NONE.
// Otherwise it will be LIBWEBP_LOSSY_ONLY for photographic images lacking an
// alpha channel and LIBWEBP_LOSSY_LOSSLESS_ALPHA otherwise. This can include
// assessments such as whether the image appears photographic or
// computer-generated, whether it actually uses its alpha channel, and
// (eventually) whether the image is animated.
// The conservative value of this field is LIBWEBP_LOSSY_ONLY, since this
// currently requries IPRO to add a Vary:Accept header to the optimized image.
optional ResourceContext.LibWebpLevel
minimal_webp_support = 22 [ default = LIBWEBP_LOSSY_ONLY ];
deprecated_minimal_webp_support = 22
[ default = LIBWEBP_LOSSY_ONLY, deprecated = true ];

// Information about any images indirectly included via this resource, or
// its dependencies.
repeated AssociatedImageInfo associated_image_info = 23;

// Does this CachedResult represent an InlineOutputResource?
// If false, this represents a plain OutputResource instead.
optional bool is_inline_output_resource = 24 [ default = false ];
optional bool is_inline_output_resource = 24;

// Type of the optimized image. This is actually net_instaweb::ImageType.
optional int32 optimized_image_type = 25;
}

// Contains the mapping of input URLs to output URLs. In the general
Expand Down Expand Up @@ -234,21 +228,28 @@ message OutputPartitions {

// Encapsulates all the data needed to rewrite a resource. Any filter needing
// additional information should add it as optional fields here.
// Next free tag: 8
// Next free tag: 9
message ResourceContext {
optional ImageDim desired_image_dims = 1;
optional bool attempt_webp = 2 [ default = false, deprecated = true ];
optional bool attempt_webp = 2 [ deprecated = true ];
optional bool inline_images = 3 [ default = true ];
optional bool mobile_user_agent = 4 [ default = false ];
optional bool mobile_user_agent = 4;

enum LibWebpLevel {
LIBWEBP_NONE = 0;
LIBWEBP_LOSSY_ONLY = 1;
LIBWEBP_LOSSY_LOSSLESS_ALPHA = 2;
LIBWEBP_ANIMATED = 3;
}

optional LibWebpLevel libwebp_level = 5 [default = LIBWEBP_NONE];

optional ImageDim user_agent_screen_resolution = 6 [deprecated = true];
optional bool use_small_screen_quality = 7 [default = false];

// Set to true if we may use the image qualities set for small screen.
optional bool may_use_small_screen_quality = 7;

// Set to true if we may use the image qualities set for save-data. The
// request must have "Save-Data: on" header to make this true. If true,
// we'll add a unique identifier to the cache key.
optional bool may_use_save_data_quality = 8;
}
15 changes: 9 additions & 6 deletions net/instaweb/rewriter/css_image_rewriter_test.cc
View file
Expand Up @@ -1084,6 +1084,9 @@ TEST_F(CssRecompressImagesInStyleAttributes,
"<div style=\"background-image:url(xfoo.jpg.pagespeed.ic.0.webp)\"/>");
}

// TODO(huibao) Remove test cases RecompressAndWebpAndStyleEnabledWithMaxCssSize
// and RecompressAndWebpLosslessAndStyleEnabledWithMaxCssSize, when
// option max_image_bytes_for_webp_in_css is removed.
TEST_F(CssRecompressImagesInStyleAttributes,
RecompressAndWebpAndStyleEnabledWithMaxCssSize) {
AddFileToMockFetcher(StrCat(kTestDomain, "foo.jpg"), kPuzzleJpgFile,
Expand All @@ -1092,12 +1095,12 @@ TEST_F(CssRecompressImagesInStyleAttributes,
options()->EnableFilter(RewriteOptions::kRecompressJpeg);
options()->EnableFilter(RewriteOptions::kRewriteStyleAttributesWithUrl);
options()->set_image_jpeg_recompress_quality(85);
options()->set_max_image_bytes_for_webp_in_css(1);
SetCurrentUserAgent("webp");
options()->set_max_image_bytes_for_webp_in_css(1); // No effect
SetupForWebp();
rewrite_driver()->AddFilters();
ValidateExpected("webp",
"<div style=\"background-image:url(foo.jpg)\"/>",
"<div style=\"background-image:url(xfoo.jpg.pagespeed.ic.0.jpg)\"/>");
"<div style=\"background-image:url(xfoo.jpg.pagespeed.ic.0.webp)\"/>");
}

TEST_F(CssRecompressImagesInStyleAttributes,
Expand All @@ -1108,12 +1111,12 @@ TEST_F(CssRecompressImagesInStyleAttributes,
options()->EnableFilter(RewriteOptions::kRecompressJpeg);
options()->EnableFilter(RewriteOptions::kRewriteStyleAttributesWithUrl);
options()->set_image_jpeg_recompress_quality(85);
options()->set_max_image_bytes_for_webp_in_css(1);
SetCurrentUserAgent("webp-la");
options()->set_max_image_bytes_for_webp_in_css(1); // No effect
SetupForWebpLossless();
rewrite_driver()->AddFilters();
ValidateExpected("webp-lossless",
"<div style=\"background-image:url(foo.jpg)\"/>",
"<div style=\"background-image:url(xfoo.jpg.pagespeed.ic.0.jpg)\"/>");
"<div style=\"background-image:url(xfoo.jpg.pagespeed.ic.0.webp)\"/>");
}

// https://code.google.com/p/modpagespeed/issues/detail?id=781
Expand Down
24 changes: 23 additions & 1 deletion net/instaweb/rewriter/device_properties.cc
View file
Expand Up @@ -31,6 +31,7 @@ DeviceProperties::DeviceProperties(UserAgentMatcher* matcher)
supports_image_inlining_(kNotSet),
supports_js_defer_(kNotSet),
supports_lazyload_images_(kNotSet),
requests_save_data_(kNotSet),
accepts_webp_(kNotSet),
supports_webp_rewritten_urls_(kNotSet),
supports_webp_lossless_alpha_(kNotSet),
Expand All @@ -40,7 +41,8 @@ DeviceProperties::DeviceProperties(UserAgentMatcher* matcher)
supports_split_html_(kNotSet),
supports_flush_early_(kNotSet),
device_type_set_(kNotSet),
device_type_(UserAgentMatcher::kDesktop) {
device_type_(UserAgentMatcher::kDesktop),
has_via_header_(kNotSet) {
}

DeviceProperties::~DeviceProperties() {
Expand Down Expand Up @@ -74,6 +76,18 @@ void DeviceProperties::ParseRequestHeaders(
request_headers.HasValue(HttpAttributes::kAcceptEncoding,
HttpAttributes::kGzip) ?
kTrue : kFalse;

const char* save_data_header =
request_headers.Lookup1(HttpAttributes::kSaveData);
if (save_data_header != NULL && StringCaseEqual("on", save_data_header)) {
requests_save_data_ = kTrue;
} else {
requests_save_data_ = kFalse;
}

has_via_header_ =
request_headers.Has(HttpAttributes::kVia) ?
kTrue : kFalse;
}

bool DeviceProperties::AcceptsGzip() const {
Expand Down Expand Up @@ -229,4 +243,12 @@ bool DeviceProperties::ForbidWebpInlining() const {
return false;
}

bool DeviceProperties::RequestsSaveData() const {
return (requests_save_data_ == kTrue);
}

bool DeviceProperties::HasViaHeader() const {
return (has_via_header_ == kTrue);
}

} // namespace net_instaweb
44 changes: 44 additions & 0 deletions net/instaweb/rewriter/device_properties_test.cc
View file
Expand Up @@ -27,6 +27,16 @@ class DevicePropertiesTest: public testing::Test {
DevicePropertiesTest()
: device_properties_(&user_agent_matcher_) { }

void ParseAndVerifySaveData(const char* header_value, bool expected_value) {
RequestHeaders headers;
if (header_value != NULL) {
headers.Add(HttpAttributes::kSaveData, header_value);
}
DeviceProperties device_properties(&user_agent_matcher_);
device_properties.ParseRequestHeaders(headers);
EXPECT_EQ(expected_value, device_properties.RequestsSaveData());
}

UserAgentMatcher user_agent_matcher_;
DeviceProperties device_properties_;
};
Expand Down Expand Up @@ -161,4 +171,38 @@ TEST_F(DevicePropertiesTest, WebpUserAgentIdentificationAccept) {
EXPECT_TRUE(device_properties_.SupportsWebpLosslessAlpha());
}

TEST_F(DevicePropertiesTest, ProcessSaveDataHeader) {
ParseAndVerifySaveData("on", true);
ParseAndVerifySaveData("oN", true);
ParseAndVerifySaveData("ON", true);
ParseAndVerifySaveData(NULL, false);
ParseAndVerifySaveData("off", false);
ParseAndVerifySaveData("ofF", false);
ParseAndVerifySaveData("", false);
ParseAndVerifySaveData("garbage", false);
}

TEST_F(DevicePropertiesTest, ProcessViaHeader) {
RequestHeaders headers1;
DeviceProperties device_properties1(&user_agent_matcher_);
// This is to verify that unrelated headers don't set HasViaHeader().
headers1.Add(HttpAttributes::kAccept, "image/webp");
headers1.Add(HttpAttributes::kAccept, "text/html");
device_properties1.ParseRequestHeaders(headers1);
EXPECT_FALSE(device_properties1.HasViaHeader());

RequestHeaders headers2;
DeviceProperties device_properties2(&user_agent_matcher_);
headers2.Add(HttpAttributes::kVia,
"1.0 fred, 1.1 example.com (Apache/1.1)");
device_properties2.ParseRequestHeaders(headers2);
EXPECT_TRUE(device_properties2.HasViaHeader());

RequestHeaders headers3;
DeviceProperties device_properties3(&user_agent_matcher_);
headers3.Add(HttpAttributes::kVia, "");
device_properties3.ParseRequestHeaders(headers3);
EXPECT_TRUE(device_properties3.HasViaHeader());
}

} // namespace net_instaweb
16 changes: 1 addition & 15 deletions net/instaweb/rewriter/image.cc
View file
Expand Up @@ -351,8 +351,7 @@ Image::Image(const StringPiece& original_contents)
original_contents_(original_contents),
output_contents_(),
output_valid_(false),
rewrite_attempted_(false),
minimal_webp_support_(ResourceContext::LIBWEBP_NONE) { }
rewrite_attempted_(false) { }

ImageImpl::ImageImpl(const StringPiece& original_contents,
const GoogleString& url,
Expand Down Expand Up @@ -833,10 +832,6 @@ bool ImageImpl::ComputeOutputContents() {
break;
case IMAGE_WEBP:
case IMAGE_WEBP_LOSSLESS_OR_ALPHA:
minimal_webp_support_ =
(image_type_ == IMAGE_WEBP) ?
ResourceContext::LIBWEBP_LOSSY_ONLY :
ResourceContext::LIBWEBP_LOSSY_LOSSLESS_ALPHA;
if (resized || options_->recompress_webp) {
ok = MayConvert() &&
ReduceWebpImageQuality(string_for_image,
Expand All @@ -851,7 +846,6 @@ bool ImageImpl::ComputeOutputContents() {
ok = false;
break;
case IMAGE_JPEG:
minimal_webp_support_ = ResourceContext::LIBWEBP_LOSSY_ONLY;
if (MayConvert() &&
options_->convert_jpeg_to_webp &&
(options_->preferred_webp != WEBP_NONE)) {
Expand All @@ -860,7 +854,6 @@ bool ImageImpl::ComputeOutputContents() {
VLOG(1) << "Image conversion: " << ok << " jpeg->webp for " << url_;
if (!ok) {
// Image is not going to be webp-converted!
minimal_webp_support_ = ResourceContext::LIBWEBP_NONE;
PS_LOG_INFO(handler_, "Failed to create webp!");
}
}
Expand Down Expand Up @@ -1053,7 +1046,6 @@ inline bool ImageImpl::ComputeOutputContentsFromGifOrPng(
if (options_->preferred_webp == WEBP_ANIMATED &&
options_->webp_animated_quality > 0) {
output_type = IMAGE_WEBP_ANIMATED;
minimal_webp_support_ = ResourceContext::LIBWEBP_ANIMATED;
}
// else we can't recompress this image
} else if (is_photo && options_->convert_png_to_jpeg &&
Expand All @@ -1062,7 +1054,6 @@ inline bool ImageImpl::ComputeOutputContentsFromGifOrPng(
// Can be converted to lossy format.
if (!has_transparency) {
// No alpha; can be converted to WebP lossy or JPEG.
minimal_webp_support_ = ResourceContext::LIBWEBP_LOSSY_ONLY;
if (options_->preferred_webp != WEBP_NONE &&
options_->convert_jpeg_to_webp &&
options_->webp_quality > 0) {
Expand Down Expand Up @@ -1090,9 +1081,6 @@ inline bool ImageImpl::ComputeOutputContentsFromGifOrPng(

if (output_type == IMAGE_WEBP_ANIMATED) {
ok = ConvertAnimatedGifToWebp(has_transparency);
if (!ok) {
minimal_webp_support_ = ResourceContext::LIBWEBP_NONE;
}
} else {
if (output_type == IMAGE_WEBP ||
output_type == IMAGE_WEBP_LOSSLESS_OR_ALPHA) {
Expand All @@ -1103,8 +1091,6 @@ inline bool ImageImpl::ComputeOutputContentsFromGifOrPng(
// TODO(huibao): Re-evaluate why we need to try a different format, if the
// conversion to WebP failed.
if (!ok) {
// We can't convert to webp at all, so register that fact.
minimal_webp_support_ = ResourceContext::LIBWEBP_NONE;
// If the conversion to WebP failed, we will try converting the image to
// jpeg or png.
if (output_type == IMAGE_WEBP) {
Expand Down

0 comments on commit 4d2000c

Please sign in to comment.