diff --git a/doomsday/apps/libdoomsday/include/doomsday/resource/databundle.h b/doomsday/apps/libdoomsday/include/doomsday/resource/databundle.h index f8ad5f6d9e..4a9673d558 100644 --- a/doomsday/apps/libdoomsday/include/doomsday/resource/databundle.h +++ b/doomsday/apps/libdoomsday/include/doomsday/resource/databundle.h @@ -64,6 +64,11 @@ class LIBDOOMSDAY_PUBLIC DataBundle : public de::IByteArray de::File const &asFile() const; de::File const &sourceFile() const; + /** + * Identifier of the package representing this data bundle (after being identified). + */ + de::String packageId() const; + /** * Generates appropriate packages according to the contents of the data bundle. */ diff --git a/doomsday/apps/libdoomsday/net.dengine.base.pack/databundles.dei b/doomsday/apps/libdoomsday/net.dengine.base.pack/databundles.dei index 706f6b52da..c98282a947 100644 --- a/doomsday/apps/libdoomsday/net.dengine.base.pack/databundles.dei +++ b/doomsday/apps/libdoomsday/net.dengine.base.pack/databundles.dei @@ -2,7 +2,8 @@ # # This file contains the identification criteria for common known data # files from the original games. Each data file is represented by a -# package with the given identifier and version. +# package with the given identifier and version. At least two of the +# criteria must match for successful identification. # # When Doomsday locates a WAD file, the lump directory CRC32 and other # information is printed in the log. These can be used when adding new diff --git a/doomsday/apps/libdoomsday/src/resource/bundles.cpp b/doomsday/apps/libdoomsday/src/resource/bundles.cpp index 7f5d1d5c6f..1baa313f2c 100644 --- a/doomsday/apps/libdoomsday/src/resource/bundles.cpp +++ b/doomsday/apps/libdoomsday/src/resource/bundles.cpp @@ -31,6 +31,8 @@ using namespace de; namespace res { +static int const MATCH_MINIMUM_SCORE = 2; + DENG2_PIMPL(Bundles) , DENG2_OBSERVES(FileIndex, Addition) { @@ -260,6 +262,12 @@ Bundles::MatchResult Bundles::match(DataBundle const &bundle) const } } + if (match.bestScore < MATCH_MINIMUM_SCORE) + { + // No go. + return MatchResult(); + } + LOG_RES_VERBOSE("Matched: %s %s %s score: %i") << match.packageId << match.packageVersion.asText() diff --git a/doomsday/apps/libdoomsday/src/resource/databundle.cpp b/doomsday/apps/libdoomsday/src/resource/databundle.cpp index 202fa60b21..e51114d563 100644 --- a/doomsday/apps/libdoomsday/src/resource/databundle.cpp +++ b/doomsday/apps/libdoomsday/src/resource/databundle.cpp @@ -29,10 +29,13 @@ #include #include #include +#include #include using namespace de; +static String const VAR_TAGS("tags"); + namespace internal { static char const *formatDescriptions[] = @@ -54,6 +57,7 @@ DENG2_PIMPL(DataBundle) SafePtr source; Format format; String packageId; // linked under /sys/bundles/ + String versionedPackageId; std::unique_ptr lumpDir; SafePtr pkgLink; @@ -70,16 +74,46 @@ DENG2_PIMPL(DataBundle) return App::rootFolder().locate("/sys/bundles"); } - void identify() + static String cleanIdentifier(String const &text) { - LOG_AS("DataBundle"); + String cleaned = text.toLower(); + cleaned.replace(QRegExp("[._]"), "-"); // periods and underscores have special meaning in packages IDs + return cleaned; + } - DENG2_ASSERT(packageId.isEmpty()); // should be only called once + static String stripVersion(String const &text, Version *version = nullptr) + { + QRegExp re(".*([-_. ][0-9._-]+)$"); + if (re.exactMatch(text)) + { + if (version) + { + String str = re.cap(1).mid(1); + str.replace("_", "."); + version->parseVersionString(str); + } + return text.mid(0, text.size() - re.cap(1).size()); + } + return text; + } + + /** + * Identifies the data bundle and sets up a package link under "/sys/bundles" with + * the appropriate metadata. + * + * Sets up the package metadata according to the best matched known information or + * autogenerated entries. + */ + void identify() + { + // It is sufficient to identify each bundle only once. if (!packageId.isEmpty()) return; // Load the lump directory of WAD files. if (format == Wad || format == Pwad || format == Iwad) { + // The lump directory needs to be loaded before matching against known + // bundles because it can be used for identification. lumpDir.reset(new res::LumpDirectory(source->as())); if (!lumpDir->isValid()) { @@ -96,22 +130,28 @@ DENG2_PIMPL(DataBundle) << "\nfileSize:" << source->size() << "\nlumpDirCRC32:" << QString::number(lumpDir->crc32(), 16).toLatin1();*/ } - else if (self.isNested()) + else if (!self.containerPackageId().isEmpty()) { - //qDebug() << "[DataBundle]" << source->description().toLatin1().constData() - // << "is nested, no package will be generated"; + // This file is inside a package, so the package will take care of it. + /*qDebug() << "[DataBundle]" << source->description().toLatin1().constData() + << "is nested, no package will be generated";*/ return; } // Search for known data files in the bundle registry. res::Bundles::MatchResult matched = DoomsdayApp::bundles().match(self); File &dataFile = self.asFile(); + String const dataFilePath = dataFile.path(); // Metadata for the package will be collected into this record. Record meta; - meta.set("path", dataFile.path()); + meta.set("path", dataFilePath); meta.set("bundleScore", matched.bestScore); + DataBundle const *container = self.containerBundle(); + + // At least two criteria must match -- otherwise simply having the correct + // type would be accepted. if (matched) { // Package metadata has been defined for this file (databundles.dei). @@ -123,24 +163,21 @@ DENG2_PIMPL(DataBundle) .value().as().setSemanticHints(NumberValue::Hex); } + meta.set("version", matched.packageVersion.asText()); meta.set("title", matched.bestMatch->keyValue("info:title")); meta.set("author", matched.bestMatch->keyValue("info:author")); meta.set("license", matched.bestMatch->keyValue("info:license", "Unknown")); - meta.set("tags", matched.bestMatch->keyValue("info:tags")); + meta.set(VAR_TAGS, matched.bestMatch->keyValue("info:tags")); } else { - // Perhaps there is metadata provided: - // - Info entry inside root folder - // - .manifest companion - - + meta.set("version", "0.0"); meta.set("title", dataFile.name()); meta.set("author", "Unknown"); meta.set("license", "Unknown"); - meta.set("tags", "generated"); + meta.set(VAR_TAGS, dataFile.extension()); - // Generate an identifier based on the information we have. + // Generate a default identifier based on the information we have. static String const formatDomains[] = { "file.local", "file.pk3", @@ -152,25 +189,66 @@ DENG2_PIMPL(DataBundle) "file.dehacked", "file.collection" }; - String cleanName = source->name().fileNameWithoutExtension().toLower(); - cleanName.replace('.', "-"); // periods have special meaning in packages IDs - packageId = formatDomains[format] + "." + cleanName; + + // Containers become part of the identifier. + for (DataBundle const *i = container; i; i = i->containerBundle()) + { + packageId = cleanIdentifier(stripVersion(i->sourceFile().name().fileNameWithoutExtension())) + .concatenateMember(packageId); + } + + // The file name may contain a version number. + Version parsedVersion(""); + String strippedName = stripVersion(source->name().fileNameWithoutExtension(), + &parsedVersion); + if (strippedName != source->name()) + { + meta.set("version", parsedVersion.asText()); + } + + packageId = formatDomains[format] + .concatenateMember(packageId) + .concatenateMember(cleanIdentifier(strippedName)); + + /* + * There may be non-native metadata available, though: + * - Info entry inside root folder + * - .manifest companion + */ + File const *sbInfo = App::rootFolder().tryLocate( + dataFilePath.fileNamePath() / dataFilePath.fileNameWithoutExtension() + + ".manifest"); + if (!sbInfo) + { + sbInfo = App::rootFolder().tryLocate(dataFilePath/"Info"); + } + if (!sbInfo) + { + sbInfo = App::rootFolder().tryLocate(dataFilePath/"Contents/Info"); + } + if (sbInfo) + { + parseSnowberryInfo(*sbInfo, meta); + } + + if (!dataFile.extension().compareWithoutCase(".box")) + { + meta.set(VAR_TAGS, meta.gets(VAR_TAGS) + " box"); + } } LOG_RES_VERBOSE("Identified \"%s\" %s %s score: %i") << packageId - << matched.packageVersion.asText() + << meta.gets("version") << ::internal::formatDescriptions[format] << matched.bestScore; - /* - * Set up the package metadata according to the best matched known - * information or autogenerated entries. - */ + + versionedPackageId = packageId; // Finally, make a link that represents the package. if (auto chosen = chooseUniqueLinkPathAndVersion(dataFile, packageId, - matched.packageVersion, - matched.bestScore)) + meta.gets("version"), + matched.bestScore)) { LOGDEV_RES_VERBOSE("Linking %s as %s") << dataFile.path() << chosen.path; @@ -181,9 +259,125 @@ DENG2_PIMPL(DataBundle) metadata.copyMembersFrom(meta); metadata.set("version", !chosen.version.isEmpty()? chosen.version : "0.0"); + // Compose a versioned ID. + if (!chosen.version.isEmpty()) + { + versionedPackageId += "_" + chosen.version; + } + LOG_RES_VERBOSE("Generated package:\n%s") << metadata.asText(); App::fileSystem().index(*pkgLink); + + // Make this a required package in the container bundle. + if (container && isAutoLoaded()) + { + // Autoloaded data files are hidden. + meta.set(VAR_TAGS, meta.gets(VAR_TAGS) + " hidden"); + + // Make sure that the container has been fully identified. + container->identifyPackages(); + + if (container->d->pkgLink) + { + qDebug() << container->d->versionedPackageId << "loads" << versionedPackageId + << "from" << dataFilePath; + Package::addRequiredPackage(*container->d->pkgLink, versionedPackageId); + } + } + } + } + + /** + * Determines if the data bundle is intended to be automatically loaded by Doomsday + * according to the v1.x autoload rules. + */ + bool isAutoLoaded() const + { + Path const path(self.asFile().path()); + + if (path.segmentCount() >= 3) + { + String const parent = path.reverseSegment(1).toString().toLower(); + String const grandParent = path.reverseSegment(2).toString().toLower(); + + if (parent.fileNameExtension() == ".pk3" || + parent.fileNameExtension() == ".box") + { + // Data files in the root of a PK3/box are all automatically loaded. + return true; + } + if (parent.fileNameExtension().isEmpty() && + (parent == "auto" || parent.beginsWith("#") || parent.beginsWith("@"))) + { + return true; + } + + if (grandParent.fileNameExtension() == ".box") + { + if (parent == "required") + { + return true; + } + /// @todo What about "Extra"? + } + } + return false; + } + + /** + * Reads a Snowberry-style Info file and extracts the relevant parts into the + * Doomsday 2 package metadata record. + * + * @param infoFile Snowberry Info file. + * @param meta Package metadata. + */ + void parseSnowberryInfo(File const &infoFile, Record &meta) + { + Info info(String::fromUtf8(Block(infoFile))); + auto const &rootBlock = info.root(); + + // Tag it as a Snowberry package. + meta.set(VAR_TAGS, meta.gets(VAR_TAGS) + " legacy"); + + if (rootBlock.contains("name")) + { + meta.set("title", rootBlock.keyValue("name")); + } + + String component = rootBlock.keyValue("component"); + if (!component.isEmpty()) + { + String gameTags; + if (!component.compareWithoutCase("game-jdoom")) + { + gameTags = " doom doom2"; + } + else if (!component.compareWithoutCase("game-jheretic")) + { + gameTags = " heretic"; + } + else if (!component.compareWithoutCase("game-jhexen")) + { + gameTags = " hexen"; + } + if (!gameTags.isEmpty()) + { + meta.set(VAR_TAGS, meta.gets(VAR_TAGS) + gameTags); + } + } + + if (Info::BlockElement const *english = rootBlock.findAs("english")) + { + if(english->blockType() == "language") + { + // Doomsday must understand the version number. + meta.set("version", Version(english->keyValue("version")).asText()); + meta.set("author", english->keyValue("author")); + meta.set("license", english->keyValue("license")); + meta.set("contact", english->keyValue("contact")); + meta.set("notes", english->keyValue("readme")); + } } } @@ -200,7 +394,7 @@ DENG2_PIMPL(DataBundle) Version const &packageVersion, dint bundleScore) { - for (int attempt = 0; attempt < 3; ++attempt) + for (int attempt = 0; attempt < 4; ++attempt) { String linkPath = packageId; String version = (packageVersion.isValid()? packageVersion.asText() : ""); @@ -211,21 +405,36 @@ DENG2_PIMPL(DataBundle) case 0: // unmodified break; - case 1: // parent folder as version label + case 1: // parse version from parent folder + case 2: // parent folder as version label if (dataFile.path().fileNamePath() != "/local/wads") { - if (version.isEmpty()) version = "0"; Path const filePath(dataFile.path()); if (filePath.segmentCount() >= 2) { - version += "-" + filePath.segment(filePath.segmentCount() - 2).toString().toLower(); + auto const &parentName = filePath.reverseSegment(1) + .toString().fileNameWithoutExtension(); + if (attempt == 1) + { + Version parsed(""); + stripVersion(parentName, &parsed); + if (parsed.isValid()) + { + version = parsed.asText(); + } + } + else + { + version = "0-" + filePath.reverseSegment(1) + .toString().fileNameWithoutExtension().toLower(); + } } } break; - case 2: // status + case 3: // status version = version.concatenateMember(dataFile.status().modifiedAt - .asDateTime().toString("yyMMdd.hhmmss")); + .asDateTime().toString("yyyyMMdd.hhmmss")); break; } @@ -247,7 +456,7 @@ DENG2_PIMPL(DataBundle) { // This could still be a better scored match. Record const &pkgInfo = bundleFolder().locate(linkPath).objectNamespace(); - if (bundleScore > pkgInfo.geti("bundleScore")) + if (bundleScore > pkgInfo.geti("package.bundleScore")) { // Forget about the previous link. bundleFolder().removeFile(linkPath); @@ -302,6 +511,15 @@ File const &DataBundle::sourceFile() const return *asFile().source(); } +String DataBundle::packageId() const +{ + if (d->packageId.isEmpty()) + { + identifyPackages(); + } + return d->packageId; +} + IByteArray::Size DataBundle::size() const { if (d->source) @@ -344,13 +562,14 @@ void DataBundle::setFormat(Format format) void DataBundle::identifyPackages() const { + LOG_AS("DataBundle"); try { d->identify(); } catch (Error const &er) { - LOG_RES_WARNING("Failed to identify %s") << description(); + LOG_RES_WARNING("Failed to identify %s: %s") << description() << er.asText(); } } @@ -367,7 +586,9 @@ DataBundle const *DataBundle::containerBundle() const for (Folder const *folder = file->parent(); folder; folder = folder->parent()) { if (auto const *data = folder->maybeAs()) + { return data; + } } return nullptr; } @@ -396,7 +617,7 @@ File *DataBundle::Interpreter::interpretFile(File *sourceData) const { ".deh", Dehacked }, { ".box", Collection }, }; - String const ext = sourceData->name().fileNameExtension(); + String const ext = sourceData->extension(); for (auto const &fmt : formats) { if (!ext.compareWithoutCase(fmt.str))