Skip to content

Commit

Permalink
Handle case collisions on case-insensitive systems
Browse files Browse the repository at this point in the history 
When running NixOps under Mac OS X, we need to be able to import store
paths built on Linux into the local Nix store. However, HFS+ is
usually case-insensitive, so if there are directories with file names
that differ only in case, then importing will fail.

The solution is to add a suffix ("~nix~case~hack~<integer>") to
colliding files. For instance, if we have a directory containing
xt_CONNMARK.h and xt_connmark.h, then the latter will be renamed to
"xt_connmark.h~nix~case~hack~1". If a store path is dumped as a NAR,
the suffixes are removed. Thus, importing and exporting via a
case-insensitive Nix store is round-tripping. So when NixOps calls
nix-copy-closure to copy the path to a Linux machine, you get the
original file names back.

Closes #119.
  • Loading branch information
@edolstra
edolstra committed Jul 16, 2014
1 parent bb65460 commit 276a40b
Show file tree
Hide file tree
Showing 6 changed files with 122 additions and 76 deletions.
2 changes: 2 additions & 0 deletions src/libstore/globals.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

#include "globals.hh"
#include "util.hh"
#include "archive.hh"

#include <map>
#include <algorithm>
Expand Down Expand Up @@ -150,6 +151,7 @@ void Settings::update()
get(useSshSubstituter, "use-ssh-substituter");
get(logServers, "log-servers");
get(enableImportNative, "allow-unsafe-native-code-during-evaluation");
get(useCaseHack, "use-case-hack");

string subs = getEnv("NIX_SUBSTITUTERS", "default");
if (subs == "default") {
Expand Down
163 changes: 92 additions & 71 deletions src/libutil/archive.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
#include <cerrno>
#include <algorithm>
#include <vector>
#include <map>
#include <cstring>

#define _XOPEN_SOURCE 600
#include <sys/types.h>
Expand All @@ -18,47 +20,29 @@
namespace nix {


bool useCaseHack =
#if __APPLE__
true;
#else
false;
#endif

static string archiveVersion1 = "nix-archive-1";

static string caseHackSuffix = "~nix~case~hack~";

PathFilter defaultPathFilter;


static void dump(const string & path, Sink & sink, PathFilter & filter);


static void dumpEntries(const Path & path, Sink & sink, PathFilter & filter)
{
Strings names = readDirectory(path);
vector<string> names2(names.begin(), names.end());
sort(names2.begin(), names2.end());

for (vector<string>::iterator i = names2.begin();
i != names2.end(); ++i)
{
Path entry = path + "/" + *i;
if (filter(entry)) {
writeString("entry", sink);
writeString("(", sink);
writeString("name", sink);
writeString(*i, sink);
writeString("node", sink);
dump(entry, sink, filter);
writeString(")", sink);
}
}
}


static void dumpContents(const Path & path, size_t size,
static void dumpContents(const Path & path, size_t size,
Sink & sink)
{
writeString("contents", sink);
writeLongLong(size, sink);

AutoCloseFD fd = open(path.c_str(), O_RDONLY);
if (fd == -1) throw SysError(format("opening file `%1%'") % path);

unsigned char buf[65536];
size_t left = size;

Expand Down Expand Up @@ -89,12 +73,41 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
writeString("", sink);
}
dumpContents(path, (size_t) st.st_size, sink);
}
}

else if (S_ISDIR(st.st_mode)) {
writeString("type", sink);
writeString("directory", sink);
dumpEntries(path, sink, filter);

/* If we're on a case-insensitive system like Mac OS X, undo
the case hack applied by restorePath(). */
Strings names = readDirectory(path);
std::map<string, string> unhacked;
for (auto & i : names)
if (useCaseHack) {
string name(i);
size_t pos = i.find(caseHackSuffix);
if (pos != string::npos) {
printMsg(lvlDebug, format("removing case hack suffix from `%1%'") % (path + "/" + i));
name.erase(pos);
}
if (unhacked.find(name) != unhacked.end())
throw Error(format("file name collision in between `%1%' and `%2%'")
% (path + "/" + unhacked[name]) % (path + "/" + i));
unhacked[name] = i;
} else
unhacked[i] = i;

for (auto & i : unhacked)
if (filter(path + "/" + i.first)) {
writeString("entry", sink);
writeString("(", sink);
writeString("name", sink);
writeString(i.first, sink);
writeString("node", sink);
dump(path + "/" + i.second, sink, filter);
writeString(")", sink);
}
}

else if (S_ISLNK(st.st_mode)) {
Expand Down Expand Up @@ -123,50 +136,21 @@ static SerialisationError badArchive(string s)
}


#if 0
static void skipGeneric(Source & source)
{
if (readString(source) == "(") {
while (readString(source) != ")")
skipGeneric(source);
}
}


static void parse(ParseSink & sink, Source & source, const Path & path);



static void parseEntry(ParseSink & sink, Source & source, const Path & path)
{
string s, name;

s = readString(source);
if (s != "(") throw badArchive("expected open tag");

while (1) {
checkInterrupt();

s = readString(source);

if (s == ")") {
break;
} else if (s == "name") {
name = readString(source);
} else if (s == "node") {
if (s == "") throw badArchive("entry name missing");
parse(sink, source, path + "/" + name);
} else {
throw badArchive("unknown field " + s);
skipGeneric(source);
}
}
}
#endif


static void parseContents(ParseSink & sink, Source & source, const Path & path)
{
unsigned long long size = readLongLong(source);

sink.preallocateContents(size);

unsigned long long left = size;
Expand All @@ -185,6 +169,15 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path)
}


struct CaseInsensitiveCompare
{
bool operator() (const string & a, const string & b) const
{
return strcasecmp(a.c_str(), b.c_str()) < 0;
}
};


static void parse(ParseSink & sink, Source & source, const Path & path)
{
string s;
Expand All @@ -194,6 +187,8 @@ static void parse(ParseSink & sink, Source & source, const Path & path)

enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown;

std::map<Path, int, CaseInsensitiveCompare> names;

while (1) {
checkInterrupt();

Expand Down Expand Up @@ -221,9 +216,9 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
else if (t == "symlink") {
type = tpSymlink;
}

else throw badArchive("unknown file type " + t);

}

else if (s == "contents" && type == tpRegular) {
Expand All @@ -236,25 +231,51 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
}

else if (s == "entry" && type == tpDirectory) {
parseEntry(sink, source, path);
string name;

s = readString(source);
if (s != "(") throw badArchive("expected open tag");

while (1) {
checkInterrupt();

s = readString(source);

if (s == ")") {
break;
} else if (s == "name") {
name = readString(source);
if (useCaseHack) {
auto i = names.find(name);
if (i != names.end()) {
printMsg(lvlDebug, format("case collision between `%1%' and `%2%'") % i->first % name);
name += caseHackSuffix;
name += int2String(++i->second);
} else
names[name] = 0;
}
} else if (s == "node") {
if (s.empty()) throw badArchive("entry name missing");
parse(sink, source, path + "/" + name);
} else
throw badArchive("unknown field " + s);
}
}

else if (s == "target" && type == tpSymlink) {
string target = readString(source);
sink.createSymlink(path, target);
}

else {
else
throw badArchive("unknown field " + s);
skipGeneric(source);
}
}
}


void parseDump(ParseSink & sink, Source & source)
{
string version;
string version;
try {
version = readString(source);
} catch (SerialisationError & e) {
Expand Down Expand Up @@ -323,13 +344,13 @@ struct RestoreSink : ParseSink
}
};


void restorePath(const Path & path, Source & source)
{
RestoreSink sink;
sink.dstPath = path;
parseDump(sink, source);
}


}
12 changes: 8 additions & 4 deletions src/libutil/archive.hh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ namespace nix {
where:
attrs(as) = concat(map(attr, as)) + encN(0)
attrs(as) = concat(map(attr, as)) + encN(0)
attrs((a, b)) = encS(a) + encS(b)
encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary)
Expand Down Expand Up @@ -58,18 +58,22 @@ void dumpPath(const Path & path, Sink & sink,
struct ParseSink
{
virtual void createDirectory(const Path & path) { };

virtual void createRegularFile(const Path & path) { };
virtual void isExecutable() { };
virtual void preallocateContents(unsigned long long size) { };
virtual void receiveContents(unsigned char * data, unsigned int len) { };

virtual void createSymlink(const Path & path, const string & target) { };
};

void parseDump(ParseSink & sink, Source & source);

void restorePath(const Path & path, Source & source);



// FIXME: global variables are bad m'kay.
extern bool useCaseHack;


}
19 changes: 19 additions & 0 deletions tests/case-hack.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
source common.sh

clearStore

rm -rf $TEST_ROOT/case

opts="--option use-case-hack true"

# Check whether restoring and dumping a NAR that contains case
# collisions is round-tripping, even on a case-insensitive system.
nix-store $opts --restore $TEST_ROOT/case < case.nar
nix-store $opts --dump $TEST_ROOT/case > $TEST_ROOT/case.nar
cmp case.nar $TEST_ROOT/case.nar
[ "$(nix-hash $opts --type sha256 $TEST_ROOT/case)" = "$(nix-hash --flat --type sha256 case.nar)" ]

# Check whether we detect true collisions (e.g. those remaining after
# removal of the suffix).
touch "$TEST_ROOT/case/xt_CONNMARK.h~nix~case~hack~3"
! nix-store $opts --dump $TEST_ROOT/case > /dev/null
Binary file added tests/case.nar
View file
Binary file not shown.
2 changes: 1 addition & 1 deletion tests/local.mk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ nix_tests = \
remote-store.sh export.sh export-graph.sh negative-caching.sh \
binary-patching.sh timeout.sh secure-drv-outputs.sh nix-channel.sh \
multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \
binary-cache.sh nix-profile.sh repair.sh dump-db.sh
binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh
# parallel.sh

install-tests += $(foreach x, $(nix_tests), tests/$(x))
Expand Down

0 comments on commit 276a40b

Please sign in to comment.