diff --git a/build b/build index a249cd2..00e6458 100755 --- a/build +++ b/build @@ -1,3 +1,3 @@ #!/bin/sh -rm -f dist/m4b-tool* bin/m4b-tool_*.log +rm -f dist/m4b-tool* bin/*.log php -d phar.readonly=off tools/box.phar build && chmod +x dist/*.phar && tar -C dist -czf dist/m4b-tool.tar.gz m4b-tool.phar && cd dist && zip m4b-tool.zip m4b-tool.phar && cd - diff --git a/src/library/M4bTool/Audio/MetaDataHandler.php b/src/library/M4bTool/Audio/MetaDataHandler.php index 87004c2..35feda5 100644 --- a/src/library/M4bTool/Audio/MetaDataHandler.php +++ b/src/library/M4bTool/Audio/MetaDataHandler.php @@ -25,6 +25,10 @@ class MetaDataHandler implements TagReaderInterface, TagWriterInterface, Duratio const FORMAT_MP3 = "mp3"; + const CODEC_MP3 = "mp3"; + const CODEC_AAC = "aac"; + const CODEC_ALAC = "alac"; + const EXTENSION_FORMAT_MAPPING = [ self::EXTENSION_M4A => self::FORMAT_MP4, self::EXTENSION_M4B => self::FORMAT_MP4, @@ -57,10 +61,6 @@ public function estimateDuration(SplFileInfo $file): ?TimeUnit return $this->ffmpeg->estimateDuration($file); } - public function detectExactDuration(SplFileInfo $file) - { - - } public function detectFormat(SplFileInfo $file) { @@ -79,10 +79,6 @@ private static function getFormatByExtension(SplFileInfo $file) } -// public function detectCodec(SplFileInfo $file) { -// -// } - public function loadTag(SplFileInfo $file) { diff --git a/src/library/M4bTool/Command/MergeCommand.php b/src/library/M4bTool/Command/MergeCommand.php index d15aa7e..f64aa38 100644 --- a/src/library/M4bTool/Command/MergeCommand.php +++ b/src/library/M4bTool/Command/MergeCommand.php @@ -15,11 +15,13 @@ use M4bTool\Audio\Chapter; use M4bTool\Audio\Silence; use M4bTool\Executables\Ffmpeg; +use M4bTool\Executables\FileConverterOptions; use M4bTool\Executables\Mp4art; use M4bTool\Executables\Mp4chaps; use M4bTool\Executables\Mp4info; use M4bTool\Executables\Mp4tags; use M4bTool\Executables\Mp4v2Wrapper; +use M4bTool\Executables\Tasks\ConversionTask; use M4bTool\Filesystem\DirectoryLoader; use M4bTool\Chapter\ChapterMarker; use M4bTool\Parser\FfmetaDataParser; @@ -49,6 +51,7 @@ class MergeCommand extends AbstractConversionCommand implements MetaReaderInterf const OPTION_NO_CONVERSION = "no-conversion"; const OPTION_BATCH_PATTERN = "batch-pattern"; const OPTION_DRY_RUN = "dry-run"; + const OPTION_JOBS = "jobs"; const MAPPING_OPTIONS_PLACEHOLDERS = [ @@ -130,6 +133,7 @@ protected function configure() $this->addOption(static::OPTION_BATCH_PATTERN, null, InputOption::VALUE_OPTIONAL | InputOption::VALUE_IS_ARRAY, "multiple batch patterns that can be used to merge all audio books in a directory matching the given patterns (e.g. %a/%t for author/title)", []); $this->addOption(static::OPTION_DRY_RUN, null, InputOption::VALUE_NONE, "perform a dry run without converting all the files in batch mode (requires --" . static::OPTION_BATCH_PATTERN . ")"); + $this->addOption(static::OPTION_JOBS, null, InputOption::VALUE_OPTIONAL, "Specifies the number of jobs (commands) to run simultaneously", 1); } @@ -603,22 +607,6 @@ private function prepareMergeWithoutConversion() } } - /** - * @return string - * @throws Exception - */ - private function createOutputTempDir() - { - $dir = $this->outputFile->getPath() ? $this->outputFile->getPath() . DIRECTORY_SEPARATOR : ""; - $dir .= $this->outputFile->getBasename("." . $this->outputFile->getExtension()) . "-tmpfiles" . DIRECTORY_SEPARATOR; - - if (!is_dir($dir) && !mkdir($dir, 0755, true)) { - $message = sprintf("Could not create temp directory %s", $dir); - $this->debug($message); - throw new Exception($message); - } - return $dir; - } /** * @throws InvalidArgumentException * @throws Exception @@ -629,50 +617,175 @@ private function convertInputFiles() $this->adjustBitrateForIpod($this->filesToConvert); $coverTargetFile = new SPLFileInfo($this->argInputFile . "/cover.jpg"); - $forceExtractCover = $this->optForce; + + $baseFdkAacCommand = $this->buildFdkaacCommand(); + $firstFile = reset($this->filesToConvert); + if ($firstFile) { + $this->extractCover($firstFile, $coverTargetFile, $this->optForce); + } + $outputTempDir = $this->createOutputTempDir(); - foreach ($this->filesToConvert as $index => $file) { - // use "force" flag only once - $this->extractCover($file, $coverTargetFile, $forceExtractCover); - $forceExtractCover = false; + if ($baseFdkAacCommand) { + foreach ($this->filesToConvert as $index => $file) { - $pad = str_pad($index + 1, $padLen, "0", STR_PAD_LEFT); - $outputFile = new SplFileInfo($outputTempDir . $pad . '-' . $file->getBasename("." . $file->getExtension()) . "-converting." . $this->optAudioExtension); - $finishedOutputFile = new SplFileInfo($outputTempDir . $pad . '-' . $file->getBasename("." . $file->getExtension()) . "-finished." . $this->optAudioExtension); + $pad = str_pad($index + 1, $padLen, "0", STR_PAD_LEFT); + $outputFile = new SplFileInfo($outputTempDir . $pad . '-' . $file->getBasename("." . $file->getExtension()) . "-converting." . $this->optAudioExtension); + $finishedOutputFile = new SplFileInfo($outputTempDir . $pad . '-' . $file->getBasename("." . $file->getExtension()) . "-finished." . $this->optAudioExtension); - $this->filesToMerge[] = $finishedOutputFile; + $this->filesToMerge[] = $finishedOutputFile; - if ($outputFile->isFile()) { - unlink($outputFile); - } + if ($outputFile->isFile()) { + unlink($outputFile); + } - if ($finishedOutputFile->isFile() && $finishedOutputFile->getSize() > 0) { - $this->notice("output file " . $outputFile . " already exists, skipping"); - continue; + if ($finishedOutputFile->isFile() && $finishedOutputFile->getSize() > 0) { + $this->notice("output file " . $outputFile . " already exists, skipping"); + continue; + } + + + if ($baseFdkAacCommand) { + $this->otherTmpFiles[] = $this->executeFdkaacCommand($baseFdkAacCommand, $file, $outputFile); + } else { + $this->executeFfmpegCommand($file, $outputFile); + } + + + if (!$outputFile->isFile()) { + throw new Exception("could not convert " . $file . " to " . $outputFile); + } + + if ($outputFile->getSize() == 0) { + unlink($outputFile); + throw new Exception("could not convert " . $file . " to " . $outputFile); + } + + rename($outputFile, $finishedOutputFile); } + } else { + $ffmpeg = new Ffmpeg(); + /** @var ConversionTask[] $conversionTasks */ + $conversionTasks = []; + foreach ($this->filesToConvert as $index => $file) { - if ($baseFdkAacCommand) { - $this->otherTmpFiles[] = $this->executeFdkaacCommand($baseFdkAacCommand, $file, $outputFile); - } else { - $this->executeFfmpegCommand($file, $outputFile); + $pad = str_pad($index + 1, $padLen, "0", STR_PAD_LEFT); + $outputFile = new SplFileInfo($outputTempDir . $pad . '-' . $file->getBasename("." . $file->getExtension()) . "-converting." . $this->optAudioExtension); + $finishedOutputFile = new SplFileInfo($outputTempDir . $pad . '-' . $file->getBasename("." . $file->getExtension()) . "-finished." . $this->optAudioExtension); + + $this->filesToMerge[] = $finishedOutputFile; + + if ($outputFile->isFile()) { + unlink($outputFile); + } + + + $options = new FileConverterOptions(); + $options->source = $file; + $options->destination = $outputFile; + $options->tempDir = $outputTempDir; + $options->extension = $this->optAudioExtension; + $options->codec = $this->optAudioCodec; + $options->format = $this->optAudioFormat; + $options->channels = $this->optAudioChannels; + $options->sampleRate = $this->optAudioSampleRate; + $options->bitRate = $this->optAudioBitRate; + $options->force = $this->optForce; + + $conversionTasks[] = new ConversionTask($ffmpeg, $options); } + $jobs = $this->input->getOption(static::OPTION_JOBS) ? (int)$this->input->getOption(static::OPTION_JOBS) : 1; + + // minimum 1 job, maximum count conversionTasks jobs + $jobs = max(min($jobs, count($conversionTasks)), 1); + + $runningTaskCount = 0; + $conversionTaskQueue = $conversionTasks; + $runningTasks = []; + $start = microtime(true); + $increaseProgressBarSeconds = 5; + do { + $firstFailedTask = null; + if ($runningTaskCount > 0 && $firstFailedTask === null) { + foreach ($runningTasks as $task) { + if ($task->didFail()) { + $firstFailedTask = $task; + break; + } + } + } + + // add new tasks, if no task did fail and jobs left + /** @var ConversionTask $task */ + $task = null; + while ($firstFailedTask === null && $runningTaskCount < $jobs && $task = array_shift($conversionTaskQueue)) { + $task->run(); + $runningTasks[] = $task; + $runningTaskCount++; + } + + usleep(250000); + + $runningTasks = array_filter($runningTasks, function (ConversionTask $task) { + return $task->isRunning(); + }); + + $runningTaskCount = count($runningTasks); + $conversionQueueLength = count($conversionTaskQueue); - if (!$outputFile->isFile()) { - throw new Exception("could not convert " . $file . " to " . $outputFile); + $time = microtime(true); + $progressBar = str_repeat("+", ceil(($time - $start) / $increaseProgressBarSeconds)); + $this->output->write(sprintf("\r%d/%d remaining tasks running: %s", $runningTaskCount, ($conversionQueueLength + $runningTaskCount), $progressBar), false, OutputInterface::VERBOSITY_VERBOSE); + + } while ($conversionQueueLength > 0 || $runningTaskCount > 0); + $this->output->writeln("", OutputInterface::VERBOSITY_VERBOSE); + /** @var ConversionTask $firstFailedTask */ + if ($firstFailedTask !== null) { + throw new Exception("a task has failed", null, $firstFailedTask->getLastException()); } - if ($outputFile->getSize() == 0) { - unlink($outputFile); - throw new Exception("could not convert " . $file . " to " . $outputFile); + + /** @var ConversionTask $task */ + foreach ($conversionTasks as $index => $task) { + $pad = str_pad($index + 1, $padLen, "0", STR_PAD_LEFT); + $file = $task->getOptions()->source; + $outputFile = $task->getOptions()->destination; + $finishedOutputFile = new SplFileInfo($outputTempDir . $pad . '-' . $file->getBasename("." . $file->getExtension()) . "-finished." . $this->optAudioExtension); + + if (!$outputFile->isFile()) { + throw new Exception("could not convert " . $file . " to " . $outputFile); + } + + if ($outputFile->getSize() == 0) { + unlink($outputFile); + throw new Exception("could not convert " . $file . " to " . $outputFile); + } + + rename($outputFile, $finishedOutputFile); } + } + + } - rename($outputFile, $finishedOutputFile); + /** + * @return string + * @throws Exception + */ + private function createOutputTempDir() + { + $dir = $this->outputFile->getPath() ? $this->outputFile->getPath() . DIRECTORY_SEPARATOR : ""; + $dir .= $this->outputFile->getBasename("." . $this->outputFile->getExtension()) . "-tmpfiles" . DIRECTORY_SEPARATOR; + + if (!is_dir($dir) && !mkdir($dir, 0755, true)) { + $message = sprintf("Could not create temp directory %s", $dir); + $this->debug($message); + throw new Exception($message); } + return $dir; } /** @@ -798,46 +911,6 @@ private function mergeFiles() return $outputTempFile; } - private function deleteTemporaryFiles() - { - if ($this->optDebug) { - return; - } - - if ($this->input->getOption(static::OPTION_NO_CONVERSION)) { - return; - } - - try { - $this->deleteFilesAndParentDir($this->filesToMerge); - $this->deleteFilesAndParentDir($this->otherTmpFiles); - } catch (Throwable $e) { - $this->error("could not delete temporary files: ", $e->getMessage()); - $this->debug("trace:", $e->getTraceAsString()); - } - - } - - private function deleteFilesAndParentDir(array $files) - { - $file = null; - foreach ($files as $file) { - unlink($file); - } - if ($file === null) { - return true; - } - $parentDir = dirname($file); - $recIt = new RecursiveDirectoryIterator($parentDir, FilesystemIterator::SKIP_DOTS); - $it = new IteratorIterator($recIt); - $filesToDelete = iterator_to_array($it); - if (count($filesToDelete) > 0) { - return false; - } - rmdir($parentDir); - return true; - } - /** * @param SplFileInfo $outputFile * @throws InvalidArgumentException @@ -909,5 +982,45 @@ private function moveFinishedOutputFile(SplFileInfo $outputTempFile, SplFileInfo } } + private function deleteTemporaryFiles() + { + if ($this->optDebug) { + return; + } + + if ($this->input->getOption(static::OPTION_NO_CONVERSION)) { + return; + } + + try { + $this->deleteFilesAndParentDir($this->filesToMerge); + $this->deleteFilesAndParentDir($this->otherTmpFiles); + } catch (Throwable $e) { + $this->error("could not delete temporary files: ", $e->getMessage()); + $this->debug("trace:", $e->getTraceAsString()); + } + + } + + private function deleteFilesAndParentDir(array $files) + { + $file = null; + foreach ($files as $file) { + unlink($file); + } + if ($file === null) { + return true; + } + $parentDir = dirname($file); + $recIt = new RecursiveDirectoryIterator($parentDir, FilesystemIterator::SKIP_DOTS); + $it = new IteratorIterator($recIt); + $filesToDelete = iterator_to_array($it); + if (count($filesToDelete) > 0) { + return false; + } + rmdir($parentDir); + return true; + } + } diff --git a/src/library/M4bTool/Executables/AbstractExecutable.php b/src/library/M4bTool/Executables/AbstractExecutable.php index 83b9559..2b76d9b 100644 --- a/src/library/M4bTool/Executables/AbstractExecutable.php +++ b/src/library/M4bTool/Executables/AbstractExecutable.php @@ -42,12 +42,18 @@ public function __construct($pathToBinary, ProcessHelper $processHelper = null, } - protected function createProcess(array $arguments, $messageInCaseOfError = null) + protected function runProcess(array $arguments, $messageInCaseOfError = null) { array_unshift($arguments, $this->pathToBinary); return $this->processHelper->run($this->output, $arguments, $messageInCaseOfError); } + protected function createNonBlockingProcess(array $arguments) + { + array_unshift($arguments, $this->pathToBinary); + return new Process($arguments); + } + protected function appendParameterToCommand(&$command, $parameterName, $parameterValue = null) { if (is_bool($parameterValue)) { diff --git a/src/library/M4bTool/Executables/Fdkaac.php b/src/library/M4bTool/Executables/Fdkaac.php new file mode 100644 index 0000000..1ae5a12 --- /dev/null +++ b/src/library/M4bTool/Executables/Fdkaac.php @@ -0,0 +1,84 @@ +createProcess($arguments); + return $this->runProcess($arguments); } /** @@ -203,27 +205,6 @@ public function loadHighestAvailableQualityAacCodec() return static::AAC_FALLBACK_CODEC; } - /** - * @param SplFileInfo $file - * @return TimeUnit|void - * @throws Exception - */ - public function estimateDuration(SplFileInfo $file): ?TimeUnit - { - $process = $this->ffmpeg([ - "-hide_banner", - "-i", $file, - "-f", "ffmetadata", - "-"]); - $output = $process->getOutput() . $process->getErrorOutput(); - - preg_match("/\bDuration:[\s]+([0-9:\.]+)/", $output, $matches); - if (!isset($matches[1])) { - return null; - } - return TimeUnit::fromFormat($matches[1], TimeUnit::FORMAT_DEFAULT); - } - /** * @param SplFileInfo $file * @return TimeUnit|void @@ -254,14 +235,25 @@ private function createStreamInfoProcess(SplFileInfo $file) ]); } - private function createMetaDataProcess(SplFileInfo $file) + /** + * @param SplFileInfo $file + * @return TimeUnit|void + * @throws Exception + */ + public function estimateDuration(SplFileInfo $file): ?TimeUnit { - return $this->ffmpeg([ + $process = $this->ffmpeg([ "-hide_banner", "-i", $file, "-f", "ffmetadata", - "-" - ]); + "-"]); + $output = $process->getOutput() . $process->getErrorOutput(); + + preg_match("/\bDuration:[\s]+([0-9:\.]+)/", $output, $matches); + if (!isset($matches[1])) { + return null; + } + return TimeUnit::fromFormat($matches[1], TimeUnit::FORMAT_DEFAULT); } /** @@ -284,6 +276,16 @@ public function readTag(SplFileInfo $file): Tag return $metaData->toTag(); } + private function createMetaDataProcess(SplFileInfo $file) + { + return $this->ffmpeg([ + "-hide_banner", + "-i", $file, + "-f", "ffmetadata", + "-" + ]); + } + public function detectSilences(SplFileInfo $file, TimeUnit $silenceLength) { $process = $this->ffmpeg([ @@ -298,4 +300,53 @@ public function detectSilences(SplFileInfo $file, TimeUnit $silenceLength) return $silenceParser->parse($this->getAllProcessOutput($process)); } + + /** + * @param FileConverterOptions $options + * @return Process + */ + public function convertFile(FileConverterOptions $options): Process + { + $inputFile = $options->source; + $command = [ + "-i", $inputFile, + "-max_muxing_queue_size", "9999", + "-map_metadata", "0", + ]; + + // backwards compatibility: ffmpeg needed experimental flag in earlier versions + if ($options->codec == MetaDataHandler::CODEC_AAC) { + $command[] = "-strict"; + $command[] = "experimental"; + } + + + // Relocating moov atom to the beginning of the file can facilitate playback before the file is completely downloaded by the client. + $command[] = "-movflags"; + $command[] = "+faststart"; + + // no video for files is required because chapters will not work if video is embedded and shorter than audio length + $command[] = "-vn"; + + $this->appendParameterToCommand($command, "-y", $options->force); + $this->appendParameterToCommand($command, "-ab", $options->bitRate); + $this->appendParameterToCommand($command, "-ar", $options->sampleRate); + $this->appendParameterToCommand($command, "-ac", $options->channels); + $this->appendParameterToCommand($command, "-acodec", $options->codec); + + // alac can be used for m4a/m4b, but not ffmpeg says it is not mp4 compilant + if ($options->format && $options->codec !== MetaDataHandler::CODEC_ALAC) { + $this->appendParameterToCommand($command, "-f", $options->format); + } + + $command[] = $options->destination; + $process = $this->createNonBlockingProcess($command); + $process->start(); + return $process; + } + + public function supportsConversion(FileConverterOptions $options): bool + { + return true; + } } \ No newline at end of file diff --git a/src/library/M4bTool/Executables/FileConverterInterface.php b/src/library/M4bTool/Executables/FileConverterInterface.php new file mode 100644 index 0000000..6efa672 --- /dev/null +++ b/src/library/M4bTool/Executables/FileConverterInterface.php @@ -0,0 +1,17 @@ +cover, $file]; // $this->appendParameterToCommand($command, "-f", $this->optForce); - $process = $this->createProcess($command); + $process = $this->runProcess($command); if ($process->getExitCode() !== 0) { throw new Exception(sprintf("Could not add cover to file: %s, %s, %d", $file, $process->getOutput() . $process->getErrorOutput(), $process->getExitCode())); diff --git a/src/library/M4bTool/Executables/Mp4chaps.php b/src/library/M4bTool/Executables/Mp4chaps.php index 856e9b0..6b47d13 100644 --- a/src/library/M4bTool/Executables/Mp4chaps.php +++ b/src/library/M4bTool/Executables/Mp4chaps.php @@ -39,7 +39,7 @@ public function writeTag(SplFileInfo $file, Tag $tag, Flags $flags = null) file_put_contents($chaptersFile, $this->chaptersToMp4v2Format($tag->chapters)); $command[] = "-i"; $command[] = $file; - $process = $this->createProcess($command); + $process = $this->runProcess($command); if ($process->getExitCode() !== 0) { unlink($chaptersFile); diff --git a/src/library/M4bTool/Executables/Mp4info.php b/src/library/M4bTool/Executables/Mp4info.php index f2e0456..529e6fe 100644 --- a/src/library/M4bTool/Executables/Mp4info.php +++ b/src/library/M4bTool/Executables/Mp4info.php @@ -19,7 +19,7 @@ public function __construct($pathToBinary = "mp4info", ProcessHelper $processHel public function estimateDuration(SplFileInfo $file): ?TimeUnit { - $process = $this->createProcess([$file]); + $process = $this->runProcess([$file]); $output = $process->getOutput() . $process->getErrorOutput(); preg_match("/([1-9][0-9]*\.[0-9]{3}) secs,/isU", $output, $matches); if (!isset($matches[1])) { diff --git a/src/library/M4bTool/Executables/Mp4tags.php b/src/library/M4bTool/Executables/Mp4tags.php index 26597a2..67de767 100644 --- a/src/library/M4bTool/Executables/Mp4tags.php +++ b/src/library/M4bTool/Executables/Mp4tags.php @@ -60,7 +60,7 @@ public function writeTag(SplFileInfo $file, Tag $tag, Flags $flags = null) } $command[] = $file; - $process = $this->createProcess($command); + $process = $this->runProcess($command); if ($process->getExitCode() !== 0) { throw new Exception(sprintf("Could not tag file: %s, %s, %d", $file, $process->getOutput() . $process->getErrorOutput(), $process->getExitCode())); @@ -74,7 +74,7 @@ public function writeTag(SplFileInfo $file, Tag $tag, Flags $flags = null) private function doesMp4tagsSupportSorting() { $command = ["-help"]; - $process = $this->createProcess($command); + $process = $this->runProcess($command); $result = $process->getOutput() . $process->getErrorOutput(); $searchStrings = ["-sortname", "-sortartist", "-sortalbum"]; foreach ($searchStrings as $searchString) { diff --git a/src/library/M4bTool/Executables/Tasks/ConversionTask.php b/src/library/M4bTool/Executables/Tasks/ConversionTask.php new file mode 100644 index 0000000..67af0a9 --- /dev/null +++ b/src/library/M4bTool/Executables/Tasks/ConversionTask.php @@ -0,0 +1,83 @@ +ffmpeg = $ffmpeg; + $this->options = $options; + $pad = uniqid("", true); + $file = $this->options->source; + $options = clone $this->options; + $options->destination = new SplFileInfo($this->options->tempDir . $pad . '-' . $file->getBasename("." . $file->getExtension()) . "-converting." . $this->options->extension); + } + + public function run() + { + try { + $this->lastException = null; + $this->process = $this->ffmpeg->convertFile($this->options); + } catch (Throwable $e) { + $this->lastException = $e; + } + + } + + public function isRunning() + { + if ($this->process) { + return $this->process->isRunning(); + } + return false; + } + + /** + * @return Process + */ + public function getProcess() + { + return $this->process; + } + + public function didFail() + { + return $this->lastException instanceof Throwable; + } + + public function getLastException() + { + return $this->lastException; + } + + public function getOptions() + { + return $this->options; + } +} \ No newline at end of file diff --git a/src/library/M4bTool/Executables/Tasks/Runnable.php b/src/library/M4bTool/Executables/Tasks/Runnable.php new file mode 100644 index 0000000..d91e719 --- /dev/null +++ b/src/library/M4bTool/Executables/Tasks/Runnable.php @@ -0,0 +1,10 @@ +