Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 522 lines (486 sloc) 20.795 kB
24e7411 @hzeller o Move SndFileHandler as ConvolveFileHandler out of folve-filesystem.cc
authored
1 // -*- c++ -*-
2 // Folve - A fuse filesystem that convolves audio files on-the-fly.
3 //
4 // Copyright (C) 2012 Henner Zeller <h.zeller@acm.org>
5 //
6 // This program is free software; you can redistribute it and/or modify
7 // it under the terms of the GNU General Public License as published by
8 // the Free Software Foundation; either version 3 of the License, or
9 // (at your option) any later version.
10 //
11 // This program is distributed in the hope that it will be useful,
12 // but WITHOUT ANY WARRANTY; without even the implied warranty of
13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 // GNU General Public License for more details.
15 //
16 // You should have received a copy of the GNU General Public License
17 // along with this program. If not, see <http://www.gnu.org/licenses/>.
18
19 #include "convolve-file-handler.h"
20
21 #include <FLAC/metadata.h>
22 #include <sndfile.h>
23 #include <string.h>
24 #include <syslog.h>
25 #include <assert.h>
26
27 #include "conversion-buffer.h"
28 #include "folve-filesystem.h"
29 #include "sound-processor.h"
30 #include "util.h"
31 #include "zita-config.h"
32
33 using folve::DLogf;
34 using folve::Appendf;
35 using folve::StringPrintf;
36
37 // Attempt to create a ConvolveFileHandler from the given file descriptor. This
38 // returns NULL if this is not a sound-file or if there is no available
39 // convolution filter configuration available.
40 // "partial_file_info" will be set to information known so far, including
41 // error message.
42 FileHandler *ConvolveFileHandler::Create(FolveFilesystem *fs,
43 int filedes, const char *fs_path,
44 const std::string &underlying_file,
45 const std::string &filter_subdir,
46 const std::string &zita_config_dir,
47 HandlerStats *partial_file_info) {
48 SF_INFO in_info;
49 memset(&in_info, 0, sizeof(in_info));
50 SNDFILE *snd = sf_open_fd(filedes, SFM_READ, &in_info, 0);
51 if (snd == NULL) {
52 DLogf("File %s: %s", underlying_file.c_str(), sf_strerror(NULL));
53 partial_file_info->message = sf_strerror(NULL);
54 return NULL;
55 }
56
57 int bits = 16;
58 if ((in_info.format & SF_FORMAT_SUBMASK) == SF_FORMAT_PCM_24) bits = 24;
59 if ((in_info.format & SF_FORMAT_SUBMASK) == SF_FORMAT_PCM_32) bits = 32;
60
61 // Remember whatever we could got to know in the partial file info.
62 Appendf(&partial_file_info->format, "%.1fkHz, %d Bit",
63 in_info.samplerate / 1000.0, bits);
64 partial_file_info->duration_seconds = in_info.frames / in_info.samplerate;
65
66 SoundProcessor *processor = fs->processor_pool()
67 ->GetOrCreate(zita_config_dir, in_info.samplerate, in_info.channels, bits,
68 &partial_file_info->message);
69 if (processor == NULL) {
70 sf_close(snd);
71 return NULL;
72 }
73 const int seconds = in_info.frames / in_info.samplerate;
74 DLogf("File %s, %.1fkHz, %d Bit, %d:%02d: filter config %s",
75 underlying_file.c_str(), in_info.samplerate / 1000.0, bits,
76 seconds / 60, seconds % 60,
77 processor->config_file().c_str());
78 return new ConvolveFileHandler(fs, fs_path, filter_subdir,
79 underlying_file, filedes, snd, in_info,
80 *partial_file_info, processor);
81 }
82
83 ConvolveFileHandler::~ConvolveFileHandler() {
84 output_buffer_->NotifyFileComplete();
85 fs_->QuitBuffering(output_buffer_); // stop working on our files.
86 Close(); // ... so that we can close them :)
87 delete output_buffer_;
88 }
89
90 int ConvolveFileHandler::Read(char *buf, size_t size, off_t offset) {
91 if (error_) return -1;
92 const off_t current_filesize = output_buffer_->FileSize();
93 const off_t read_horizon = offset + size;
94 // If this is a skip suspiciously at the very end of the file as
95 // reported by stat, we don't do any encoding, just return garbage.
96 // (otherwise we'd to convolve up to that point).
97 //
98 // While indexing, media players do this sometimes apparently.
99 // And sometimes not even to the very end but 'almost' at the end.
100 // So add some FudeOverhang
101 static const int kFudgeOverhang = 512;
102 // But of course only if this is really a skip, not a regular approaching
103 // end-of-file.
104 if (current_filesize < offset
105 && (int) (read_horizon + kFudgeOverhang) >= file_stat_.st_size) {
106 const int pretended_bytes = std::min((off_t)size,
107 file_stat_.st_size - offset);
108 if (pretended_bytes > 0) {
109 memset(buf, 0x00, pretended_bytes);
110 return pretended_bytes;
111 } else {
112 return 0;
113 }
114 }
115 // The following read might block and call WriteToSoundfile() until the
116 // buffer is filled.
117 int result = output_buffer_->Read(buf, size, offset);
118
119 // Only if the user obviously read beyond our header, we start the
120 // pre-buffering; otherwise things will get sluggish because any header
121 // access that goes a bit overboard triggers pre-buffer (i.e. while indexing)
122 // Amarok for instance seems to read up to 16k beyond the header.
123 //
124 // In general, this is a fine heuristic: this happens the first time we
125 // access the file, the first or second stream-read should trigger the
126 // pre-buffering (64k is less than a second music). If we're in gapless
127 // mode, we already start pre-buffering anyway early (see
128 // NotifyPassedProcessorUnreferenced()) - so that important use-case is
129 // covered.
130 const off_t well_beyond_header = output_buffer_->HeaderSize() + (64 << 10);
131 const bool should_request_prebuffer = !output_buffer_->IsFileComplete()
132 && read_horizon > well_beyond_header
133 && read_horizon + fs_->pre_buffer_size() > current_filesize;
134 if (should_request_prebuffer) {
135 fs_->RequestPrebuffer(output_buffer_);
136 }
137 return result;
138 }
139
140 void ConvolveFileHandler::GetHandlerStatus(HandlerStats *stats) {
141 const off_t file_size = output_buffer_->FileSize();
142 const off_t max_access = output_buffer_->MaxAccessed();
143 if (processor_ != NULL) {
144 base_stats_.max_output_value = processor_->max_output_value();
145 }
146 *stats = base_stats_;
147 const int frames_done = in_info_.frames - frames_left();
148 if (frames_done == 0 || in_info_.frames == 0) {
149 stats->buffer_progress = 0.0;
150 stats->access_progress = 0.0;
151 } else {
152 stats->buffer_progress = 1.0 * frames_done / in_info_.frames;
153 stats->access_progress = stats->buffer_progress * max_access / file_size;
154 }
155
156 if (base_stats_.max_output_value > 1.0) {
157 // TODO: the status server could inspect this value and make better
158 // rendering.
159 base_stats_.message =
160 StringPrintf("Output clipping! "
161 "(max=%.3f; Multiply gain with <= %.5f<br/>in %s)",
162 base_stats_.max_output_value,
163 1.0 / base_stats_.max_output_value,
164 processor_ != NULL
165 ? processor_->config_file().c_str()
166 : "filter");
167 }
168 }
169
170 int ConvolveFileHandler::Stat(struct stat *st) {
171 const off_t current_file_size = output_buffer_->FileSize();
189625c @hzeller o Write a warning when file size exceeds prediction.
authored
172 if (output_buffer_->IsFileComplete()) {
173 // Once we're complete, we know exactly the final size.
174 file_stat_.st_size = current_file_size;
175 }
176 else if (current_file_size > start_estimating_size_) {
24e7411 @hzeller o Move SndFileHandler as ConvolveFileHandler out of folve-filesystem.cc
authored
177 const int frames_done = in_info_.frames - frames_left();
178 if (frames_done > 0) {
179 const float estimated_end = 1.0 * in_info_.frames / frames_done;
180 off_t new_size = estimated_end * current_file_size;
181 // Report a bit bigger size which is less harmful than programs
182 // reading short.
183 new_size += 65535;
184 if (new_size > file_stat_.st_size) { // Only go forward in size.
185 file_stat_.st_size = new_size;
186 }
187 }
188 }
189 *st = file_stat_;
190 return 0;
191 }
192
193 // TODO(hzeller): trim parameter list.
194 ConvolveFileHandler::ConvolveFileHandler(FolveFilesystem *fs,
195 const char *fs_path,
196 const std::string &filter_dir,
197 const std::string &underlying_file,
198 int filedes, SNDFILE *snd_in,
199 const SF_INFO &in_info,
200 const HandlerStats &file_info,
201 SoundProcessor *processor)
202 : FileHandler(filter_dir), fs_(fs),
203 filedes_(filedes), snd_in_(snd_in), in_info_(in_info),
204 base_stats_(file_info),
205 error_(false), output_buffer_(NULL),
206 snd_out_(NULL), processor_(processor),
207 input_frames_left_(in_info.frames) {
208
209 // Initial stat that we're going to report to clients. We'll adapt
210 // the filesize as we see it grow. Some clients continuously monitor
211 // the size of the file to check when to stop.
212 fstat(filedes_, &file_stat_);
213 start_estimating_size_ = 0.4 * file_stat_.st_size;
189625c @hzeller o Write a warning when file size exceeds prediction.
authored
214 original_file_size_ = file_stat_.st_size;
24e7411 @hzeller o Move SndFileHandler as ConvolveFileHandler out of folve-filesystem.cc
authored
215 file_stat_.st_size *= fs->file_oversize_factor();
216
217 // The flac header we get is more rich than what we can create via
218 // sndfile. So if we have one, just copy it.
219 copy_flac_header_verbatim_ = LooksLikeInputIsFlac(in_info, filedes);
220
221 // Create a conversion buffer that creates a soundfile of a particular
222 // format that we choose here. Essentially we want to generate mostly what
223 // our input is.
224 SF_INFO out_info = in_info;
225 out_info.seekable = 0;
226 if ((in_info.format & SF_FORMAT_TYPEMASK) == SF_FORMAT_OGG) {
227 // If the input was ogg, we're re-coding this to flac, because it
228 // wouldn't let us stream the output.
229 out_info.format = SF_FORMAT_FLAC;
230 out_info.format |= SF_FORMAT_PCM_16;
231 }
232 else if ((in_info.format & SF_FORMAT_TYPEMASK) == SF_FORMAT_WAV) {
233 out_info.format = SF_FORMAT_FLAC; // recode as flac.
234 out_info.format |= SF_FORMAT_PCM_24;
235 }
236 else { // original format.
237 out_info.format = in_info.format;
238 }
239
240 output_buffer_ = new ConversionBuffer(this, out_info);
241 }
242
243 void ConvolveFileHandler::SetOutputSoundfile(ConversionBuffer *out_buffer,
244 SNDFILE *sndfile) {
245 snd_out_ = sndfile;
246 if (snd_out_ == NULL) {
247 error_ = true;
248 syslog(LOG_ERR, "Opening output: %s", sf_strerror(NULL));
249 base_stats_.message = sf_strerror(NULL);
250 return;
251 }
252 if (copy_flac_header_verbatim_) {
253 out_buffer->set_sndfile_writes_enabled(false);
254 CopyFlacHeader(out_buffer);
255 } else {
256 out_buffer->set_sndfile_writes_enabled(true);
257 GenerateHeaderFromInputFile(out_buffer);
258 }
259 // Now flush the header: that way if someone only reads the metadata, then
260 // our AddMoreSoundData() is never called.
261 // We need to do this even if we copied our own header: that way we make
262 // sure that the sndfile-header is flushed into the nirwana before we
263 // re-enable sndfile_writes.
264 sf_command(snd_out_, SFC_UPDATE_HEADER_NOW, NULL, 0);
265
266 // -- time for some hackery ...
267 // If we have copied the header over from the original, we need to
268 // redact the values for min/max blocksize and min/max framesize with
269 // what SNDFILE is going to use, otherwise programs will trip over this.
270 // http://flac.sourceforge.net/format.html
271 if (copy_flac_header_verbatim_) {
272 out_buffer->WriteCharAt((1152 & 0xFF00) >> 8, 8);
273 out_buffer->WriteCharAt((1152 & 0x00FF) , 9);
274 out_buffer->WriteCharAt((1152 & 0xFF00) >> 8, 10);
275 out_buffer->WriteCharAt((1152 & 0x00FF) , 11);
276 for (int i = 12; i < 18; ++i) out_buffer->WriteCharAt(0, i); // framesize
277 } else {
278 // .. and if SNDFILE writes the header, it misses out in writing the
279 // number of samples to be expected. So let's fill that in.
280 // The MD5 sum starts at position strlen("fLaC") + 4 + 18 = 26
281 // The 32 bits before that are the samples (and another 4 bit before that,
282 // ignoring that for now).
283 out_buffer->WriteCharAt((in_info_.frames & 0xFF000000) >> 24, 22);
284 out_buffer->WriteCharAt((in_info_.frames & 0x00FF0000) >> 16, 23);
285 out_buffer->WriteCharAt((in_info_.frames & 0x0000FF00) >> 8, 24);
286 out_buffer->WriteCharAt((in_info_.frames & 0x000000FF), 25);
287 }
288
289 out_buffer->set_sndfile_writes_enabled(true); // ready for sound-stream.
290 DLogf("Header init done (%s).", base_stats_.filename.c_str());
291 out_buffer->HeaderFinished();
292 }
293
294 bool ConvolveFileHandler::HasStarted() {
295 return in_info_.frames != input_frames_left_;
296 }
297
298 bool ConvolveFileHandler::PassoverProcessor(SoundProcessor *passover_processor) {
299 if (HasStarted()) {
300 DLogf("Gapless attempt: Cannot bridge gap to already open file %s",
301 base_stats_.filename.c_str());
302 return false;
303 }
304 assert(processor_);
305 if (passover_processor->config_file() != processor_->config_file()
306 || (passover_processor->config_file_timestamp()
307 != processor_->config_file_timestamp())) {
308 DLogf("Gapless: Configuration changed; can't use %p to join gapless.",
309 passover_processor);
310 return false;
311 }
312 // Ok, so don't use the processor we already have, but use the other one.
313 fs_->processor_pool()->Return(processor_);
314 processor_ = passover_processor;
315 if (!processor_->is_input_buffer_complete()) {
316 // Fill with our beginning so that the donor can finish its processing.
317 input_frames_left_ -= processor_->FillBuffer(snd_in_);
318 }
319 base_stats_.in_gapless = true;
320 return true;
321 }
322
323 void ConvolveFileHandler::NotifyPassedProcessorUnreferenced() {
324 // This is gapless. Good idea to pre-buffer the beginning.
325 fs_->RequestPrebuffer(output_buffer_);
326 }
327
328 static bool ExtractDirAndSuffix(const std::string &filename,
329 std::string *dir, std::string *suffix) {
330 const std::string::size_type slash_pos = filename.find_last_of('/');
331 if (slash_pos == std::string::npos) return false;
332 *dir = filename.substr(0, slash_pos + 1);
333 const std::string::size_type dot_pos = filename.find_last_of('.');
334 if (dot_pos != std::string::npos && dot_pos > slash_pos) {
335 *suffix = filename.substr(dot_pos);
336 }
337 return true;
338 }
339
340 bool ConvolveFileHandler::AddMoreSoundData() {
341 if (!input_frames_left_)
342 return false;
343 if (processor_->pending_writes() > 0) {
344 processor_->WriteProcessed(snd_out_, processor_->pending_writes());
345 return input_frames_left_;
346 }
347 const int r = processor_->FillBuffer(snd_in_);
348 if (r == 0) {
349 syslog(LOG_ERR, "Expected %d frames left, "
350 "but got EOF; corrupt file '%s' ?",
351 input_frames_left_, base_stats_.filename.c_str());
352 base_stats_.message = "Premature EOF in input file.";
353 input_frames_left_ = 0;
354 Close();
355 return false;
356 }
357 stats_mutex_.Lock();
358 input_frames_left_ -= r;
359 stats_mutex_.Unlock();
360 if (!input_frames_left_ && !processor_->is_input_buffer_complete()
361 && fs_->gapless_processing()) {
362 typedef std::set<std::string> DirSet;
363 DirSet dirset;
364 std::string fs_dir, file_suffix;
365 FileHandler *next_file = NULL;
366 DirSet::const_iterator found;
367 const bool passed_processor
368 = (ExtractDirAndSuffix(base_stats_.filename, &fs_dir, &file_suffix)
369 && fs_->ListDirectory(fs_dir, file_suffix, &dirset)
370 && (found = dirset.upper_bound(base_stats_.filename)) != dirset.end()
371 && (next_file = fs_->GetOrCreateHandler(found->c_str()))
372 && next_file->PassoverProcessor(processor_));
373 if (passed_processor) {
374 DLogf("Processor %p: Gapless pass-on from "
375 "'%s' to alphabetically next '%s'", processor_,
376 base_stats_.filename.c_str(), found->c_str());
377 }
378 processor_->WriteProcessed(snd_out_, r);
379 if (passed_processor) {
380 base_stats_.out_gapless = true;
381 SaveOutputValues();
382 processor_ = NULL; // we handed over ownership.
383 Close(); // make sure that our thread is done.
384 next_file->NotifyPassedProcessorUnreferenced();
385 }
386 if (next_file) fs_->Close(found->c_str(), next_file);
387 } else {
388 processor_->WriteProcessed(snd_out_, r);
389 }
390 if (input_frames_left_ == 0) {
391 Close();
392 }
393 return input_frames_left_;
394 }
395
396 // TODO add as a utility function to ConversionBuffer ?
397 static void CopyBytes(int fd, off_t pos, ConversionBuffer *out, size_t len) {
398 char buf[256];
399 while (len > 0) {
400 ssize_t r = pread(fd, buf, std::min(sizeof(buf), len), pos);
401 if (r <= 0) return;
402 out->Append(buf, r);
403 len -= r;
404 pos += r;
405 }
406 }
407
408 void ConvolveFileHandler::CopyFlacHeader(ConversionBuffer *out_buffer) {
409 DLogf("Provide FLAC header from original file %s",
410 base_stats_.filename.c_str());
411 out_buffer->Append("fLaC", 4);
412 off_t pos = 4;
413 unsigned char header[4];
414 bool need_finish_padding = false;
415 while (pread(filedes_, header, sizeof(header), pos) == sizeof(header)) {
416 pos += sizeof(header);
417 bool is_last = header[0] & 0x80;
418 unsigned int type = header[0] & 0x7F;
419 unsigned int byte_len = (header[1] << 16) + (header[2] << 8) + header[3];
420 const char *extra_info = "";
421 need_finish_padding = false;
422 if (type == FLAC__METADATA_TYPE_STREAMINFO && byte_len == 34) {
423 out_buffer->Append(&header, sizeof(header));
424 // Copy everything but the MD5 at the end - which we set to empty.
425 CopyBytes(filedes_, pos, out_buffer, byte_len - 16);
426 for (int i = 0; i < 16; ++i) out_buffer->Append("\0", 1);
427 extra_info = "Streaminfo; redact MD5.";
428 }
429 else if (type == FLAC__METADATA_TYPE_SEEKTABLE) {
430 // The SEEKTABLE header we skip, because it is bogus after encoding.
431 // TODO append log (skip the seektable)
432 need_finish_padding = is_last; // if we were last, force finish block.
433 extra_info = "Skip seektable.";
434 }
435 else {
436 out_buffer->Append(&header, sizeof(header));
437 CopyBytes(filedes_, pos, out_buffer, byte_len);
438 }
439 DLogf(" %02x %02x %02x %02x type: %d, len: %6u %s %s ",
440 header[0], header[1], header[2], header[3],
441 type, byte_len, is_last ? "(last)" : "(cont)", extra_info);
442 pos += byte_len;
443 if (is_last)
444 break;
445 }
446 if (need_finish_padding) { // if the last block was not is_last: pad.
447 DLogf("write padding");
448 memset(&header, 0, sizeof(header));
449 header[0] = 0x80 /* is last */ | FLAC__METADATA_TYPE_PADDING;
450 out_buffer->Append(&header, sizeof(header));
451 }
452 }
453
454 void ConvolveFileHandler::GenerateHeaderFromInputFile(
455 ConversionBuffer *out_buffer) {
456 DLogf("Generate header from original ID3-tags.");
457 out_buffer->set_sndfile_writes_enabled(true);
458 // Copy ID tags that are supported by sndfile.
459 for (int i = SF_STR_FIRST; i <= SF_STR_LAST; ++i) {
460 const char *s = sf_get_string(snd_in_, i);
461 if (s != NULL) {
462 sf_set_string(snd_out_, i, s);
463 }
464 }
465 }
466
467 void ConvolveFileHandler::SaveOutputValues() {
468 if (processor_) {
469 base_stats_.max_output_value = processor_->max_output_value();
470 processor_->ResetMaxValues();
471 }
472 }
473
474 void ConvolveFileHandler::Close() {
475 if (snd_out_ == NULL) return; // done.
476 input_frames_left_ = 0;
477 SaveOutputValues();
478 if (base_stats_.max_output_value > 1.0) {
479 syslog(LOG_ERR, "Observed output clipping in '%s': "
480 "Max=%.3f; Multiply gain with <= %.5f in %s",
481 base_stats_.filename.c_str(), base_stats_.max_output_value,
482 1.0 / base_stats_.max_output_value,
483 processor_ != NULL ? processor_->config_file().c_str() : "filter");
484 }
485 fs_->processor_pool()->Return(processor_);
486 processor_ = NULL;
487 // We can't disable buffer writes here, because outfile closing will flush
488 // the last couple of sound samples.
489 if (snd_in_) sf_close(snd_in_);
490 if (snd_out_) sf_close(snd_out_);
491 snd_out_ = NULL;
492 close(filedes_);
189625c @hzeller o Write a warning when file size exceeds prediction.
authored
493
494 const double factor = 1.0 * output_buffer_->FileSize() / original_file_size_;
495 if (factor > fs_->file_oversize_factor()) {
496 syslog(LOG_WARNING, "File larger than prediction: "
d119a0a @hzeller o gap difference of off_t in fprintf() on different systems.
authored
497 "%lldx%.2f=%lld < %lld (x%4.2f) '%s'; "
189625c @hzeller o Write a warning when file size exceeds prediction.
authored
498 "naive streamer implementations might trip.",
d119a0a @hzeller o gap difference of off_t in fprintf() on different systems.
authored
499 (long long)original_file_size_, fs_->file_oversize_factor(),
500 (long long)(original_file_size_ * fs_->file_oversize_factor()),
501 (long long)output_buffer_->FileSize(), factor,
502 base_stats_.filename.c_str());
189625c @hzeller o Write a warning when file size exceeds prediction.
authored
503 }
24e7411 @hzeller o Move SndFileHandler as ConvolveFileHandler out of folve-filesystem.cc
authored
504 }
505
506 bool ConvolveFileHandler::LooksLikeInputIsFlac(const SF_INFO &sndinfo,
507 int filedes) {
508 if ((sndinfo.format & SF_FORMAT_TYPEMASK) != SF_FORMAT_FLAC)
509 return false;
510 // However some files contain flac encoded stuff, but are not flac files
511 // by themselve. So we can't copy headers verbatim. Sanity check header.
512 char flac_magic[4];
513 if (pread(filedes, flac_magic, sizeof(flac_magic), 0) != sizeof(flac_magic))
514 return false;
515 return memcmp(flac_magic, "fLaC", sizeof(flac_magic)) == 0;
516 }
517
518 int ConvolveFileHandler::frames_left() {
519 folve::MutexLock l(&stats_mutex_);
520 return input_frames_left_;
521 }
Something went wrong with that request. Please try again.