forked from dirkmc/press
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Compressor.java
567 lines (474 loc) · 20.7 KB
/
Compressor.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
package press;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileFilter;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import play.Play;
import play.PlayPlugin;
import play.cache.Cache;
import play.exceptions.UnexpectedException;
import play.libs.Crypto;
import play.mvc.Router;
import play.mvc.Http.Request;
import play.mvc.Http.Response;
import play.mvc.Router.ActionDefinition;
import play.templates.JavaExtensions;
import play.vfs.VirtualFile;
public abstract class Compressor extends PlayPlugin {
static final String PRESS_SIGNATURE = "press-1.0";
static final String PATTERN_TEXT = "^/\\*" + PRESS_SIGNATURE + "\\|(.*?)\\*/$";
static final Pattern HEADER_PATTERN = Pattern.compile(PATTERN_TEXT);
// File type, eg "JavaScript"
String fileType;
// File extension, eg ".js"
String extension;
// Play Action to get the compressed file, eg "Press.getCompressedJS"
String getCompressedFileAction;
// Tag name, eg "#{press.script}"
String tagName;
// Compressed file tag name, eg "#{press.compressed-script}"
String compressedTagName;
// Signatures for the start and end of a request to compress a file,
// eg "<!-- press js: " and " -->" would result in a compress request like:
// <!-- press js: /public/javascript/myfile.js -->
String pressRequestStart;
String pressRequestEnd;
// Directory where the source files are read from, eg "/public/javascripts"
String srcDir;
// Directory for the compressed output, eg "/public/javascripts/js"
String compressedDir;
// The key used to identify this request
String requestKey = null;
// The list of files compressed as part of this request
Map<String, FileInfo> fileInfos;
protected interface FileCompressor {
public void compress(String fileName, Reader in, Writer out) throws Exception;
}
public Compressor(String fileType, String extension, String getCompressedFileAction,
String tagName, String compressedTagName, String pressRequestStart,
String pressRequestEnd, String srcDir, String compressedDir) {
this.fileInfos = new HashMap<String, FileInfo>();
this.fileType = fileType;
this.extension = extension;
this.getCompressedFileAction = getCompressedFileAction;
this.pressRequestStart = pressRequestStart;
this.pressRequestEnd = pressRequestEnd;
this.tagName = tagName;
this.compressedTagName = compressedTagName;
this.srcDir = PluginConfig.addTrailingSlash(srcDir);
this.compressedDir = PluginConfig.addTrailingSlash(compressedDir);
}
/**
* Adds a file to the list of files to be compressed
*
* @return the file request signature to be output in the HTML
*/
public String add(String fileName, boolean compress) {
if (compress) {
PressLogger.trace("Adding %s to output", fileName);
} else {
PressLogger.trace("Adding %s to output (will not be compressed)", fileName);
}
if (fileInfos.containsKey(fileName)) {
String msg = "Attempt to add the same " + fileType + " file ";
msg += "to compression twice: '" + fileName + "'\n";
msg += "Check that you're not including the same file in two different ";
msg += tagName + " tags";
throw new PressException(msg);
}
fileInfos.put(fileName, new FileInfo(fileName, compress, null));
return getFileRequestSignature(fileName);
}
public String compressedUrl() {
if (requestKey != null) {
String msg = "There is more than one " + compressedTagName
+ " tag in the template output. " + "There must be one only.";
throw new PressException(msg);
}
requestKey = getRequestKey() + extension;
Map<String, Object> params = new HashMap<String, Object>();
params.put("key", requestKey);
ActionDefinition route = Router.reverse(getCompressedFileAction, params);
int numFiles = fileInfos.size();
PressLogger.trace("Adding key %s for compression of %d files", requestKey, numFiles);
return route.url;
}
public void saveFileList() {
// If the request key has not been set, that means there was no request
// for compressed source anywhere in the template file, so we don't
// need to generate anything
if (requestKey == null) {
// If the file list is not empty, then there have been files added
// to compression but they will not be output. So throw an
// exception telling the user he needs to add some files.
if (fileInfos.size() > 0) {
String msg = fileInfos.size() + " files added to compression with ";
msg += tagName + " tag but no " + compressedTagName + " tag was found. ";
msg += "You must include a " + compressedTagName + " tag in the template ";
msg += "to output the compressed content of these files: ";
msg += JavaExtensions.join(fileInfos.keySet(), ", ");
throw new PressException(msg);
}
return;
}
// The press tag may not always been executed by the template engine
// in the same order that the resulting <script> tags would appear in
// the HMTL output. So here we scan the output to figure out in what
// order the <script> tags should actually be output.
long timeStart = System.currentTimeMillis();
List<FileInfo> orderedFileNames = getFileListOrder();
long timeAfter = System.currentTimeMillis();
PressLogger.trace("Time to scan response for %s files for '%s': %d milli-seconds",
fileType, Request.current().url, (timeAfter - timeStart));
// Add the list of files to the cache.
// When the server receives a request for the compressed file, it will
// retrieve the list of files and compress them.
addFileListToCache(requestKey, orderedFileNames);
}
public List<FileInfo> getFileListOrder() {
String content = getResponseContent();
List<String> namesInOrder = getFilesInResponse(content);
List<FileInfo> filesInOrder = new ArrayList<FileInfo>(namesInOrder.size());
// Do some sanity checking
if (namesInOrder.size() != fileInfos.size()) {
String msg = "Number of file compress requests found in response ";
msg += "(" + namesInOrder.size() + ")";
msg += "not equal to number of files added to compression ";
msg += "(" + fileInfos.size() + "). ";
msg += "Please disable the press plugin and report a bug.";
throw new PressException(msg);
}
// Copy the FileInfo from the map into an array, in order
for (String fileName : namesInOrder) {
if (!fileInfos.containsKey(fileName)) {
String msg = "File compress request for '" + fileName + "' ";
msg += "found in response but file was never added to file list. ";
msg += "Please disable the press plugin and report a bug.";
throw new PressException(msg);
}
filesInOrder.add(fileInfos.get(fileName));
}
return filesInOrder;
}
public void addFileListToCache(String cacheKey, List<FileInfo> originalList) {
List<FileInfo> newList = new ArrayList<FileInfo>();
for (FileInfo fileInfo : originalList) {
VirtualFile file = getVirtualFile(srcDir + fileInfo.fileName);
// Check the file exists
if (!file.exists()) {
String msg = "Attempt to add file '" + file.getRealFile().getAbsolutePath() + "' ";
msg += "to compression with " + tagName + " tag but file does not exist.";
throw new PressException(msg);
}
newList.add(new FileInfo(fileInfo.fileName, fileInfo.compress, file));
}
// Add a mapping between the request key and the list of files that
// are compressed for the request
Cache.set(cacheKey, newList, PluginConfig.compressionKeyStorageTime);
}
/**
* The request key is just the url - for the same url we should always
* return the same compressed javascript or css.
*/
private String getRequestKey() {
String key = Request.current().path + Request.current().querystring + extension;
// Get a hash of the url to keep it short
String hashed = Crypto.passwordHash(key);
return lettersOnly(hashed);
}
@SuppressWarnings("unchecked")
protected static VirtualFile getCompressedFile(FileCompressor compressor, String key,
String compressedDir, String extension) {
List<FileInfo> componentFiles = (List<FileInfo>) Cache.get(key);
// If there was nothing found for the given request key, return null.
// This shouldn't happen unless there was a very long delay between the
// template being rendered and the compressed file being requested
if (componentFiles == null) {
return null;
}
return getCompressedFile(compressor, componentFiles, compressedDir, extension);
}
protected static VirtualFile getCompressedFile(FileCompressor compressor,
List<FileInfo> componentFiles, String compressedDir, String extension) {
String joinedFileNames = JavaExtensions.join(FileInfo.getFileNames(componentFiles), "");
String fileName = Crypto.passwordHash(joinedFileNames);
fileName = lettersOnly(fileName);
String filePath = compressedDir + fileName + extension;
VirtualFile file = getVirtualFile(filePath);
// If the file already exists in the cache, return it
if (useCache(componentFiles, file, extension)) {
String absolutePath = file.getRealFile().getAbsolutePath();
if (file.exists()) {
PressLogger.trace("Using existing compressed file %s", absolutePath);
return file;
} else {
PressLogger.trace("Compressed file %s does not yet exist", absolutePath);
}
}
PressLogger.trace("Generating compressed file %s from %d component files", file.getName(),
componentFiles.size());
// Create the directory if it doesn't already exist
VirtualFile dir = getVirtualFile(compressedDir);
if (!dir.exists()) {
if (!dir.getRealFile().mkdirs()) {
throw new PressException("Could not create directory for compressed file output "
+ compressedDir);
}
}
Writer out = null;
try {
// Compress the component files and write the output to a temporary
// file
File tmp = File.createTempFile("prs", extension);
out = new BufferedWriter(new FileWriter(tmp));
// Add the last modified dates of each component file to the start
// of the compressed file so that we can later check if any of them
// have changed.
String lastModifiedDates = createFileHeader(componentFiles);
out.append(lastModifiedDates);
long timeStart = System.currentTimeMillis();
for (FileInfo componentFile : componentFiles) {
compress(compressor, componentFile, out);
}
long timeAfter = System.currentTimeMillis();
PressLogger.trace("Time to compress files for '%s': %d milli-seconds", file
.getRealFile().getName(), (timeAfter - timeStart));
// Once the compressed output has been written to the temporary
// file, move it to the cache directory.
tmp.renameTo(file.getRealFile());
PressLogger.trace("Compressed file generation complete:");
PressLogger.trace(file.relativePath());
} catch (Exception e) {
throw new UnexpectedException(e);
} finally {
if (out != null) {
try {
out.close();
} catch (IOException e) {
throw new UnexpectedException(e);
}
}
}
return file;
}
public static List<File> clearCache(String compressedDir, String extension) {
PressLogger.trace("Deleting cached files");
// Get the cache directory
VirtualFile dir = getVirtualFile(compressedDir);
if (!dir.exists() || !dir.isDirectory()) {
return new ArrayList<File>();
}
// Get a list of all compressed files, and delete them
FileFilter compressedFileFilter = new PressFileFilter(extension);
File[] files = dir.getRealFile().listFiles(compressedFileFilter);
List<File> deleted = new ArrayList<File>(files.length);
for (File file : files) {
if (file.delete()) {
deleted.add(file);
}
}
PressLogger.trace("Deleted %d cached files", deleted.size());
return deleted;
}
private static boolean useCache(List<FileInfo> componentFiles, VirtualFile file,
String extension) {
PressLogger.trace("Caching strategy is %s", PluginConfig.cache);
if (PluginConfig.cache.equals(CachingStrategy.Never)) {
return false;
}
if (PluginConfig.cache.equals(CachingStrategy.Always)) {
return true;
}
boolean changed = haveComponentFilesChanged(componentFiles, file);
if (changed) {
PressLogger.trace("Component %s files have changed", extension);
} else {
PressLogger.trace("Component %s files have not changed", extension);
}
return !changed;
}
private static boolean haveComponentFilesChanged(List<FileInfo> componentFiles, VirtualFile file) {
// Check if the file exists
if (!file.exists()) {
return true;
}
// Check if the file has a compression header
String header = extractHeaderContent(file.getRealFile());
if (header == null) {
return true;
}
// Check if the number of files has changed
String[] lastModifieds = header.split(":");
if (lastModifieds.length != componentFiles.size()) {
return true;
}
try {
// Check each of the stored last modified dates against the file's
// current last modified date
for (int i = 0; i < componentFiles.size(); i++) {
FileInfo fileInfo = componentFiles.get(i);
// Check if the file was compressed and is now uncompressed,
// or vice versa
char compress = fileInfo.compress ? 'c' : 'u';
if (lastModifieds[i].charAt(0) != compress) {
return true;
}
// Check the timestamp
String lastMod = lastModifieds[i].substring(1);
long lastModified = Long.parseLong(lastMod);
if (fileInfo.file.lastModified() != lastModified) {
return true;
}
}
} catch (Exception e) {
// If there's any sort of problem reading the header, we can just
// overwrite the file
return true;
}
return false;
}
/**
* <pre>
* The file header consists of
* - An opening comment
* - A signature
* - A '|' character used as a separator
* - A ':' separated list of
* o the character 'c' or 'u', indicating whether the file is compressed
* o a unix timestamp
* - A closing comment
*
* eg
* press-1.0|c12323123:u1231212:c1312312:c1312423
* </pre>
*
*/
public static String createFileHeader(List<FileInfo> componentFiles) {
List<String> timestamps = new ArrayList<String>(componentFiles.size());
for (int i = 0; i < componentFiles.size(); i++) {
FileInfo fileInfo = componentFiles.get(i);
String lastMod = fileInfo.file.lastModified().toString();
char compress = fileInfo.compress ? 'c' : 'u';
timestamps.add(compress + lastMod);
}
return "/*" + PRESS_SIGNATURE + "|" + JavaExtensions.join(timestamps, ":") + "*/\n";
}
public static String extractHeaderContent(File file) {
try {
BufferedReader reader = new BufferedReader(new FileReader(file));
String firstLine = reader.readLine();
Matcher matcher = HEADER_PATTERN.matcher(firstLine);
if (matcher.matches()) {
return matcher.group(1);
}
return null;
} catch (IOException e) {
throw new UnexpectedException(e);
}
}
private static void compress(FileCompressor compressor, FileInfo fileInfo, Writer out)
throws Exception {
String fileName = fileInfo.file.getName();
BufferedReader in = new BufferedReader(new FileReader(fileInfo.file.getRealFile()));
// If the file should be compressed
if (fileInfo.compress) {
// Invoke the compressor
PressLogger.trace("Compressing %s", fileName);
compressor.compress(fileName, in, out);
} else {
// Otherwise just copy it
PressLogger.trace("Adding already compressed file %s", fileName);
write(in, out);
compressor.compress(fileName, in, out);
}
}
public static void write(Reader reader, Writer writer) throws IOException {
int read = 0;
char[] buffer = new char[8096];
while ((read = reader.read(buffer)) > 0) {
writer.write(buffer, 0, read);
}
}
/**
* Converts any non-letter characters in the given string to letters
*/
private static String lettersOnly(String hashed) {
char[] chars = hashed.toCharArray();
for (int i = 0; i < chars.length; i++) {
if (!Character.isLetter(chars[i])) {
chars[i] = (char) ((int) 'A' + (int) chars[i] % 26);
}
}
return new String(chars);
}
/**
* Get the content of the response sent to the client as a String
*/
protected static String getResponseContent() {
try {
String content = (String) Request.current().args.get("responseString");
if (content == null) {
content = Response.current.get().out.toString("utf-8");
Request.current().args.put("responseString", content);
}
return content;
} catch (UnsupportedEncodingException e) {
throw new UnexpectedException(e);
}
}
protected String getFileRequestSignature(String fileName) {
return pressRequestStart + fileName + pressRequestEnd;
}
protected List<String> getFilesInResponse(String content) {
List<String> filesInOrder = new ArrayList<String>();
int startIndex = content.indexOf(pressRequestStart);
while (startIndex != -1) {
int endIndex = content.indexOf(pressRequestEnd, startIndex);
if (endIndex == -1) {
return filesInOrder;
}
int fileNameStartIndex = startIndex + pressRequestStart.length();
String foundFileName = content.substring(fileNameStartIndex, endIndex);
filesInOrder.add(foundFileName);
startIndex = content.indexOf(pressRequestStart, endIndex);
}
return filesInOrder;
}
/**
* Gets the file at the given path, relative to the application root, even
* if the file doesn't exist
*/
private static VirtualFile getVirtualFile(String filePath) {
return VirtualFile.open(Play.getFile(filePath));
}
protected static class FileInfo {
String fileName;
boolean compress;
private VirtualFile file;
public FileInfo(String fileName, boolean compress, VirtualFile file) {
this.fileName = fileName;
this.compress = compress;
this.file = file;
}
public static Collection<String> getFileNames(List<FileInfo> list) {
Collection<String> fileNames = new ArrayList<String>(list.size());
for (FileInfo fileInfo : list) {
fileNames.add(fileInfo.fileName);
}
return fileNames;
}
}
}