summaryrefslogtreecommitdiff
path: root/nix
diff options
context:
space:
mode:
Diffstat (limited to 'nix')
-rw-r--r--nix/guix-register/guix-register.cc16
-rw-r--r--nix/libstore/build.cc462
-rw-r--r--nix/libstore/gc.cc75
-rw-r--r--nix/libstore/globals.cc92
-rw-r--r--nix/libstore/globals.hh26
-rw-r--r--nix/libstore/local-store.cc111
-rw-r--r--nix/libstore/local-store.hh21
-rw-r--r--nix/libstore/optimise-store.cc17
-rw-r--r--nix/libstore/pathlocks.cc2
-rw-r--r--nix/libstore/remote-store.cc48
-rw-r--r--nix/libstore/remote-store.hh34
-rw-r--r--nix/libstore/store-api.hh28
-rw-r--r--nix/libstore/worker-protocol.hh3
-rw-r--r--nix/libutil/archive.cc171
-rw-r--r--nix/libutil/archive.hh12
-rw-r--r--nix/libutil/hash.cc101
-rw-r--r--nix/libutil/serialise.cc27
-rw-r--r--nix/libutil/serialise.hh11
-rw-r--r--nix/libutil/types.hh17
-rw-r--r--nix/libutil/util.cc216
-rw-r--r--nix/libutil/util.hh36
-rw-r--r--nix/nix-daemon/guix-daemon.cc126
-rw-r--r--nix/nix-daemon/nix-daemon.cc159
23 files changed, 1047 insertions, 764 deletions
diff --git a/nix/guix-register/guix-register.cc b/nix/guix-register/guix-register.cc
index f5c610fde0..16dae62b3d 100644
--- a/nix/guix-register/guix-register.cc
+++ b/nix/guix-register/guix-register.cc
@@ -1,5 +1,5 @@
/* GNU Guix --- Functional package management for GNU
- Copyright (C) 2013, 2014 Ludovic Courtès <ludo@gnu.org>
+ Copyright (C) 2013, 2014, 2015 Ludovic Courtès <ludo@gnu.org>
Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012,
2013 Eelco Dolstra <eelco.dolstra@logicblox.com>
@@ -192,13 +192,21 @@ register_validity (LocalStore *store, std::istream &input,
store's '.links' directory, which means 'optimisePath' would try to link
to that instead of linking to the target store. Thus, disable
deduplication in this case. */
- if (optimize && prefix.empty ())
+ if (optimize)
{
/* Make sure deduplication is enabled. */
settings.autoOptimiseStore = true;
- foreach (ValidPathInfos::const_iterator, i, infos)
- store->optimisePath (i->path);
+ std::string store_dir = settings.nixStore;
+
+ /* 'optimisePath' creates temporary links under 'settings.nixStore' and
+ this must be the real target store, under PREFIX, to avoid
+ cross-device links. Thus, temporarily switch the value of
+ 'settings.nixStore'. */
+ settings.nixStore = prefix + store_dir;
+ for (auto&& i: infos)
+ store->optimisePath (prefix + i.path);
+ settings.nixStore = store_dir;
}
}
diff --git a/nix/libstore/build.cc b/nix/libstore/build.cc
index f38cd29940..85a818ba94 100644
--- a/nix/libstore/build.cc
+++ b/nix/libstore/build.cc
@@ -38,6 +38,9 @@
#if HAVE_SYS_MOUNT_H
#include <sys/mount.h>
#endif
+#if HAVE_SYS_SYSCALL_H
+#include <sys/syscall.h>
+#endif
#if HAVE_SCHED_H
#include <sched.h>
#endif
@@ -48,7 +51,7 @@
#include <linux/fs.h>
#endif
-#define CHROOT_ENABLED HAVE_CHROOT && HAVE_UNSHARE && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_PRIVATE) && defined(CLONE_NEWNS)
+#define CHROOT_ENABLED HAVE_CHROOT && HAVE_UNSHARE && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_PRIVATE) && defined(CLONE_NEWNS) && defined(SYS_pivot_root)
#if CHROOT_ENABLED
#include <sys/socket.h>
@@ -57,9 +60,8 @@
#include <netinet/ip.h>
#endif
-#if HAVE_SYS_PERSONALITY_H
+#if __linux__
#include <sys/personality.h>
-#define CAN_DO_LINUX32_BUILDS
#endif
#if HAVE_STATVFS
@@ -85,8 +87,12 @@ class Goal;
typedef std::shared_ptr<Goal> GoalPtr;
typedef std::weak_ptr<Goal> WeakGoalPtr;
+struct CompareGoalPtrs {
+ bool operator() (const GoalPtr & a, const GoalPtr & b);
+};
+
/* Set of goals. */
-typedef set<GoalPtr> Goals;
+typedef set<GoalPtr, CompareGoalPtrs> Goals;
typedef list<WeakGoalPtr> WeakGoals;
/* A map of paths to goals (and the other way around). */
@@ -173,11 +179,20 @@ public:
(important!), etc. */
virtual void cancel(bool timeout) = 0;
+ virtual string key() = 0;
+
protected:
void amDone(ExitCode result);
};
+bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) {
+ string s1 = a->key();
+ string s2 = b->key();
+ return s1 < s2;
+}
+
+
/* A mapping used to remember for each child process to what goal it
belongs, and file descriptors for receiving log data and output
path creation commands. */
@@ -238,6 +253,9 @@ public:
failure). */
bool permanentFailure;
+ /* Set if at least one derivation had a timeout. */
+ bool timedOut;
+
LocalStore & store;
std::shared_ptr<HookInstance> hook;
@@ -301,6 +319,7 @@ public:
void addToWeakGoals(WeakGoals & goals, GoalPtr p)
{
// FIXME: necessary?
+ // FIXME: O(n)
foreach (WeakGoals::iterator, i, goals)
if (i->lock() == p) return;
goals.push_back(p);
@@ -374,8 +393,6 @@ void Goal::trace(const format & f)
/* Common initialisation performed in child processes. */
static void commonChildInit(Pipe & logPipe)
{
- restoreAffinity();
-
/* Put the child in a separate session (and thus a separate
process group) so that it has no controlling terminal (meaning
that e.g. ssh cannot open /dev/tty) and it doesn't receive
@@ -400,19 +417,6 @@ static void commonChildInit(Pipe & logPipe)
close(fdDevNull);
}
-
-/* Convert a string list to an array of char pointers. Careful: the
- string list should outlive the array. */
-const char * * strings2CharPtrs(const Strings & ss)
-{
- const char * * arr = new const char * [ss.size() + 1];
- const char * * p = arr;
- foreach (Strings::const_iterator, i, ss) *p++ = i->c_str();
- *p = 0;
- return arr;
-}
-
-
/* Restore default handling of SIGPIPE, otherwise some programs will
randomly say "Broken pipe". */
static void restoreSIGPIPE()
@@ -590,7 +594,9 @@ HookInstance::HookInstance()
{
debug("starting build hook");
- Path buildHook = absPath(getEnv("NIX_BUILD_HOOK"));
+ Path buildHook = getEnv("NIX_BUILD_HOOK");
+ if (string(buildHook, 0, 1) != "/") buildHook = settings.nixLibexecDir + "/nix/" + buildHook;
+ buildHook = canonPath(buildHook);
/* Create a pipe to get the output of the child. */
fromHook.create();
@@ -602,44 +608,30 @@ HookInstance::HookInstance()
builderOut.create();
/* Fork the hook. */
- pid = maybeVfork();
- switch (pid) {
-
- case -1:
- throw SysError("unable to fork");
+ pid = startProcess([&]() {
- case 0:
- try { /* child */
+ commonChildInit(fromHook);
- commonChildInit(fromHook);
+ if (chdir("/") == -1) throw SysError("changing into `/");
- if (chdir("/") == -1) throw SysError("changing into `/");
+ /* Dup the communication pipes. */
+ if (dup2(toHook.readSide, STDIN_FILENO) == -1)
+ throw SysError("dupping to-hook read side");
- /* Dup the communication pipes. */
- if (dup2(toHook.readSide, STDIN_FILENO) == -1)
- throw SysError("dupping to-hook read side");
+ /* Use fd 4 for the builder's stdout/stderr. */
+ if (dup2(builderOut.writeSide, 4) == -1)
+ throw SysError("dupping builder's stdout/stderr");
- /* Use fd 4 for the builder's stdout/stderr. */
- if (dup2(builderOut.writeSide, 4) == -1)
- throw SysError("dupping builder's stdout/stderr");
+ execl(buildHook.c_str(), buildHook.c_str(), settings.thisSystem.c_str(),
+ (format("%1%") % settings.maxSilentTime).str().c_str(),
+ (format("%1%") % settings.printBuildTrace).str().c_str(),
+ (format("%1%") % settings.buildTimeout).str().c_str(),
+ NULL);
- execl(buildHook.c_str(), buildHook.c_str(), settings.thisSystem.c_str(),
- (format("%1%") % settings.maxSilentTime).str().c_str(),
- (format("%1%") % settings.printBuildTrace).str().c_str(),
- (format("%1%") % settings.buildTimeout).str().c_str(),
- NULL);
+ throw SysError(format("executing `%1%'") % buildHook);
+ });
- throw SysError(format("executing `%1%'") % buildHook);
-
- } catch (std::exception & e) {
- writeToStderr("build hook error: " + string(e.what()) + "\n");
- }
- _exit(1);
- }
-
- /* parent */
pid.setSeparatePG(true);
- pid.setKillSignal(SIGTERM);
fromHook.writeSide.close();
toHook.readSide.close();
}
@@ -648,7 +640,8 @@ HookInstance::HookInstance()
HookInstance::~HookInstance()
{
try {
- pid.kill();
+ toHook.writeSide.close();
+ pid.kill(true);
} catch (...) {
ignoreException();
}
@@ -761,7 +754,7 @@ private:
typedef void (DerivationGoal::*GoalState)();
GoalState state;
- /* Stuff we need to pass to initChild(). */
+ /* Stuff we need to pass to runChild(). */
typedef map<Path, Path> DirsInChroot; // maps target path to source path
DirsInChroot dirsInChroot;
typedef map<string, string> Environment;
@@ -784,17 +777,21 @@ private:
outputs to allow hard links between outputs. */
InodesSeen inodesSeen;
- /* Magic exit code denoting that setting up the child environment
- failed. (It's possible that the child actually returns the
- exit code, but ah well.) */
- const static int childSetupFailed = 189;
-
public:
DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal);
~DerivationGoal();
void cancel(bool timeout);
+ string key()
+ {
+ /* Ensure that derivations get built in order of their name,
+ i.e. a derivation named "aardvark" always comes before
+ "baboon". And substitution goals always happen before
+ derivation goals (due to "b$"). */
+ return "b$" + storePathToName(drvPath) + "$" + drvPath;
+ }
+
void work();
Path getDrvPath()
@@ -821,8 +818,8 @@ private:
/* Start building a derivation. */
void startBuilder();
- /* Initialise the builder's process. */
- void initChild();
+ /* Run the builder's process. */
+ void runChild();
friend int childEntry(void *);
@@ -879,13 +876,9 @@ DerivationGoal::~DerivationGoal()
{
/* Careful: we should never ever throw an exception from a
destructor. */
- try {
- killChild();
- deleteTmpDir(false);
- closeLogFile();
- } catch (...) {
- ignoreException();
- }
+ try { killChild(); } catch (...) { ignoreException(); }
+ try { deleteTmpDir(false); } catch (...) { ignoreException(); }
+ try { closeLogFile(); } catch (...) { ignoreException(); }
}
@@ -956,6 +949,11 @@ void DerivationGoal::init()
/* The first thing to do is to make sure that the derivation
exists. If it doesn't, it may be created through a
substitute. */
+ if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) {
+ haveDerivation();
+ return;
+ }
+
addWaitee(worker.makeSubstitutionGoal(drvPath));
state = &DerivationGoal::haveDerivation;
@@ -1209,7 +1207,7 @@ static string get(const StringPairs & map, const string & key)
static bool canBuildLocally(const string & platform)
{
return platform == settings.thisSystem
-#ifdef CAN_DO_LINUX32_BUILDS
+#if __linux__
|| (platform == "i686-linux" && settings.thisSystem == "x86_64-linux")
#endif
;
@@ -1433,9 +1431,6 @@ void DerivationGoal::buildDone()
if (pathExists(chrootRootDir + *i))
rename((chrootRootDir + *i).c_str(), i->c_str());
- if (WIFEXITED(status) && WEXITSTATUS(status) == childSetupFailed)
- throw Error(format("failed to set up the build environment for `%1%'") % drvPath);
-
if (diskFull)
printMsg(lvlError, "note: build failure may have been caused by lack of free disk space");
@@ -1469,37 +1464,41 @@ void DerivationGoal::buildDone()
outputLocks.unlock();
} catch (BuildError & e) {
- printMsg(lvlError, e.msg());
+ if (!hook)
+ printMsg(lvlError, e.msg());
outputLocks.unlock();
buildUser.release();
- /* When using a build hook, the hook will return a remote
- build failure using exit code 100. Anything else is a hook
- problem. */
- bool hookError = hook &&
- (!WIFEXITED(status) || WEXITSTATUS(status) != 100);
+ if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) {
+ if (settings.printBuildTrace)
+ printMsg(lvlError, format("@ build-failed %1% - timeout") % drvPath);
+ worker.timedOut = true;
+ }
- if (settings.printBuildTrace) {
- if (hook && hookError)
+ else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) {
+ if (settings.printBuildTrace)
printMsg(lvlError, format("@ hook-failed %1% - %2% %3%")
% drvPath % status % e.msg());
- else
+ }
+
+ else {
+ if (settings.printBuildTrace)
printMsg(lvlError, format("@ build-failed %1% - %2% %3%")
% drvPath % 1 % e.msg());
+ worker.permanentFailure = !fixedOutput && !diskFull;
+
+ /* Register the outputs of this build as "failed" so we
+ won't try to build them again (negative caching).
+ However, don't do this for fixed-output derivations,
+ since they're likely to fail for transient reasons
+ (e.g., fetchurl not being able to access the network).
+ Hook errors (like communication problems with the
+ remote machine) shouldn't be cached either. */
+ if (settings.cacheFailure && !fixedOutput && !diskFull)
+ foreach (DerivationOutputs::iterator, i, drv.outputs)
+ worker.store.registerFailedPath(i->second.path);
}
- /* Register the outputs of this build as "failed" so we won't
- try to build them again (negative caching). However, don't
- do this for fixed-output derivations, since they're likely
- to fail for transient reasons (e.g., fetchurl not being
- able to access the network). Hook errors (like
- communication problems with the remote machine) shouldn't
- be cached either. */
- if (settings.cacheFailure && !hookError && !fixedOutput)
- foreach (DerivationOutputs::iterator, i, drv.outputs)
- worker.store.registerFailedPath(i->second.path);
-
- worker.permanentFailure = !hookError && !fixedOutput && !diskFull;
amDone(ecFailed);
return;
}
@@ -1603,7 +1602,7 @@ void chmod_(const Path & path, mode_t mode)
int childEntry(void * arg)
{
- ((DerivationGoal *) arg)->initChild();
+ ((DerivationGoal *) arg)->runChild();
return 1;
}
@@ -1750,37 +1749,11 @@ void DerivationGoal::startBuilder()
/* Change ownership of the temporary build directory. */
if (chown(tmpDir.c_str(), buildUser.getUID(), buildUser.getGID()) == -1)
- throw SysError(format("cannot change ownership of `%1%'") % tmpDir);
+ throw SysError(format("cannot change ownership of '%1%'") % tmpDir);
+ }
- /* Check that the Nix store has the appropriate permissions,
- i.e., owned by root and mode 1775 (sticky bit on so that
- the builder can create its output but not mess with the
- outputs of other processes). */
- struct stat st;
- if (stat(settings.nixStore.c_str(), &st) == -1)
- throw SysError(format("cannot stat `%1%'") % settings.nixStore);
- if (!(st.st_mode & S_ISVTX) ||
- ((st.st_mode & S_IRWXG) != S_IRWXG) ||
- (st.st_gid != buildUser.getGID()))
- throw Error(format(
- "builder does not have write permission to `%2%'; "
- "try `chgrp %1% %2%; chmod 1775 %2%'")
- % buildUser.getGID() % settings.nixStore);
- }
-
-
- /* Are we doing a chroot build? Note that fixed-output
- derivations are never done in a chroot, mainly so that
- functions like fetchurl (which needs a proper /etc/resolv.conf)
- work properly. Purity checking for fixed-output derivations
- is somewhat pointless anyway. */
useChroot = settings.useChroot;
- if (fixedOutput) useChroot = false;
-
- /* Hack to allow derivations to disable chroot builds. */
- if (get(drv.env, "__noChroot") == "1") useChroot = false;
-
if (useChroot) {
#if CHROOT_ENABLED
/* Create a temporary directory in which we set up the chroot
@@ -1795,6 +1768,12 @@ void DerivationGoal::startBuilder()
printMsg(lvlChatty, format("setting up chroot environment in `%1%'") % chrootRootDir);
+ if (mkdir(chrootRootDir.c_str(), 0750) == -1)
+ throw SysError(format("cannot create ‘%1%’") % chrootRootDir);
+
+ if (chown(chrootRootDir.c_str(), 0, buildUser.getGID()) == -1)
+ throw SysError(format("cannot change ownership of ‘%1%’") % chrootRootDir);
+
/* Create a writable /tmp in the chroot. Many builders need
this. (Of course they should really respect $TMPDIR
instead.) */
@@ -1821,16 +1800,20 @@ void DerivationGoal::startBuilder()
% (buildUser.enabled() ? buildUser.getGID() : getgid())).str());
/* Create /etc/hosts with localhost entry. */
- writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n");
+ if (!fixedOutput)
+ writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n");
/* Bind-mount a user-configurable set of directories from the
host file system. */
- foreach (StringSet::iterator, i, settings.dirsInChroot) {
- size_t p = i->find('=');
+ PathSet dirs = tokenizeString<StringSet>(settings.get("build-chroot-dirs", string(DEFAULT_CHROOT_DIRS)));
+ PathSet dirs2 = tokenizeString<StringSet>(settings.get("build-extra-chroot-dirs", string("")));
+ dirs.insert(dirs2.begin(), dirs2.end());
+ for (auto & i : dirs) {
+ size_t p = i.find('=');
if (p == string::npos)
- dirsInChroot[*i] = *i;
+ dirsInChroot[i] = i;
else
- dirsInChroot[string(*i, 0, p)] = string(*i, p + 1);
+ dirsInChroot[string(i, 0, p)] = string(i, p + 1);
}
dirsInChroot[tmpDir] = tmpDir;
@@ -1841,8 +1824,12 @@ void DerivationGoal::startBuilder()
can be bind-mounted). !!! As an extra security
precaution, make the fake Nix store only writable by the
build user. */
- createDirs(chrootRootDir + settings.nixStore);
- chmod_(chrootRootDir + settings.nixStore, 01777);
+ Path chrootStoreDir = chrootRootDir + settings.nixStore;
+ createDirs(chrootStoreDir);
+ chmod_(chrootStoreDir, 01775);
+
+ if (chown(chrootStoreDir.c_str(), 0, buildUser.getGID()) == -1)
+ throw SysError(format("cannot change ownership of ‘%1%’") % chrootStoreDir);
foreach (PathSet::iterator, i, inputPaths) {
struct stat st;
@@ -1951,14 +1938,17 @@ void DerivationGoal::startBuilder()
*/
#if CHROOT_ENABLED
if (useChroot) {
- char stack[32 * 1024];
- pid = clone(childEntry, stack + sizeof(stack) - 8,
- CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWNET | CLONE_NEWIPC | CLONE_NEWUTS | SIGCHLD, this);
+ char stack[32 * 1024];
+ int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | SIGCHLD;
+ if (!fixedOutput) flags |= CLONE_NEWNET;
+ pid = clone(childEntry, stack + sizeof(stack) - 8, flags, this);
+ if (pid == -1)
+ throw SysError("cloning builder process");
} else
#endif
{
pid = fork();
- if (pid == 0) initChild();
+ if (pid == 0) runChild();
}
if (pid == -1) throw SysError("unable to fork");
@@ -1969,22 +1959,31 @@ void DerivationGoal::startBuilder()
worker.childStarted(shared_from_this(), pid,
singleton<set<int> >(builderOut.readSide), true, true);
+ /* Check if setting up the build environment failed. */
+ string msg = readLine(builderOut.readSide);
+ if (!msg.empty()) throw Error(msg);
+
if (settings.printBuildTrace) {
printMsg(lvlError, format("@ build-started %1% - %2% %3%")
% drvPath % drv.platform % logFile);
}
+
}
-void DerivationGoal::initChild()
+void DerivationGoal::runChild()
{
/* Warning: in the child we should absolutely not make any SQLite
calls! */
- bool inSetup = true;
-
try { /* child */
+ _writeToStderr = 0;
+
+ restoreAffinity();
+
+ commonChildInit(builderOut);
+
#if CHROOT_ENABLED
if (useChroot) {
/* Initialise the loopback interface. */
@@ -2001,9 +2000,11 @@ void DerivationGoal::initChild()
/* Set the hostname etc. to fixed values. */
char hostname[] = "localhost";
- sethostname(hostname, sizeof(hostname));
+ if (sethostname(hostname, sizeof(hostname)) == -1)
+ throw SysError("cannot set host name");
char domainname[] = "(none)"; // kernel default
- setdomainname(domainname, sizeof(domainname));
+ if (setdomainname(domainname, sizeof(domainname)) == -1)
+ throw SysError("cannot set domain name");
/* Make all filesystems private. This is necessary
because subtrees may have been mounted as "shared"
@@ -2021,12 +2022,17 @@ void DerivationGoal::initChild()
throw SysError(format("unable to make filesystem `%1%' private") % fs);
}
+ /* Bind-mount chroot directory to itself, to treat it as a
+ different filesystem from /, as needed for pivot_root. */
+ if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1)
+ throw SysError(format("unable to bind mount ‘%1%’") % chrootRootDir);
+
/* Set up a nearly empty /dev, unless the user asked to
bind-mount the host /dev. */
+ Strings ss;
if (dirsInChroot.find("/dev") == dirsInChroot.end()) {
createDirs(chrootRootDir + "/dev/shm");
createDirs(chrootRootDir + "/dev/pts");
- Strings ss;
ss.push_back("/dev/full");
#ifdef __linux__
if (pathExists("/dev/kvm"))
@@ -2037,13 +2043,24 @@ void DerivationGoal::initChild()
ss.push_back("/dev/tty");
ss.push_back("/dev/urandom");
ss.push_back("/dev/zero");
- foreach (Strings::iterator, i, ss) dirsInChroot[*i] = *i;
createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd");
createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin");
createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout");
createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr");
}
+ /* Fixed-output derivations typically need to access the
+ network, so give them access to /etc/resolv.conf and so
+ on. */
+ if (fixedOutput) {
+ ss.push_back("/etc/resolv.conf");
+ ss.push_back("/etc/nsswitch.conf");
+ ss.push_back("/etc/services");
+ ss.push_back("/etc/hosts");
+ }
+
+ for (auto & i : ss) dirsInChroot[i] = i;
+
/* Bind-mount all the directories from the "host"
filesystem that we want in the chroot
environment. */
@@ -2088,30 +2105,41 @@ void DerivationGoal::initChild()
throw SysError("mounting /dev/pts");
createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx");
- /* Make sure /dev/pts/ptmx is world-writable. With some
- Linux versions, it is created with permissions 0. */
- chmod_(chrootRootDir + "/dev/pts/ptmx", 0666);
+ /* Make sure /dev/pts/ptmx is world-writable. With some
+ Linux versions, it is created with permissions 0. */
+ chmod_(chrootRootDir + "/dev/pts/ptmx", 0666);
}
- /* Do the chroot(). Below we do a chdir() to the
- temporary build directory to make sure the current
- directory is in the chroot. (Actually the order
- doesn't matter, since due to the bind mount tmpDir and
- tmpRootDit/tmpDir are the same directories.) */
- if (chroot(chrootRootDir.c_str()) == -1)
- throw SysError(format("cannot change root directory to `%1%'") % chrootRootDir);
+ /* Do the chroot(). */
+ if (chdir(chrootRootDir.c_str()) == -1)
+ throw SysError(format("cannot change directory to '%1%'") % chrootRootDir);
+
+ if (mkdir("real-root", 0) == -1)
+ throw SysError("cannot create real-root directory");
+
+#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
+ if (pivot_root(".", "real-root") == -1)
+ throw SysError(format("cannot pivot old root directory onto '%1%'") % (chrootRootDir + "/real-root"));
+#undef pivot_root
+
+ if (chroot(".") == -1)
+ throw SysError(format("cannot change root directory to '%1%'") % chrootRootDir);
+
+ if (umount2("real-root", MNT_DETACH) == -1)
+ throw SysError("cannot unmount real root filesystem");
+
+ if (rmdir("real-root") == -1)
+ throw SysError("cannot remove real-root directory");
}
#endif
- commonChildInit(builderOut);
-
if (chdir(tmpDir.c_str()) == -1)
throw SysError(format("changing into `%1%'") % tmpDir);
/* Close all other file descriptors. */
closeMostFDs(set<int>());
-#ifdef CAN_DO_LINUX32_BUILDS
+#if __linux__
/* Change the personality to 32-bit if we're doing an
i686-linux build on an x86_64-linux machine. */
struct utsname utsbuf;
@@ -2119,7 +2147,7 @@ void DerivationGoal::initChild()
if (drv.platform == "i686-linux" &&
(settings.thisSystem == "x86_64-linux" ||
(!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) {
- if (personality(0x0008 | 0x8000000 /* == PER_LINUX32_3GB */) == -1)
+ if (personality(PER_LINUX32) == -1)
throw SysError("cannot set i686-linux personality");
}
@@ -2129,17 +2157,18 @@ void DerivationGoal::initChild()
int cur = personality(0xffffffff);
if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */);
}
+
+ /* Disable address space randomization for improved
+ determinism. */
+ int cur = personality(0xffffffff);
+ if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE);
#endif
/* Fill in the environment. */
Strings envStrs;
foreach (Environment::const_iterator, i, env)
envStrs.push_back(rewriteHashes(i->first + "=" + i->second, rewritesToTmp));
- const char * * envArr = strings2CharPtrs(envStrs);
-
- Path program = drv.builder.c_str();
- std::vector<const char *> args; /* careful with c_str()! */
- string user; /* must be here for its c_str()! */
+ auto envArr = stringsToCharPtrs(envStrs);
/* If we are running in `build-users' mode, then switch to the
user we allocated above. Make sure that we drop all root
@@ -2165,23 +2194,26 @@ void DerivationGoal::initChild()
}
/* Fill in the arguments. */
+ Strings args;
string builderBasename = baseNameOf(drv.builder);
- args.push_back(builderBasename.c_str());
+ args.push_back(builderBasename);
foreach (Strings::iterator, i, drv.args)
- args.push_back(rewriteHashes(*i, rewritesToTmp).c_str());
- args.push_back(0);
+ args.push_back(rewriteHashes(*i, rewritesToTmp));
+ auto argArr = stringsToCharPtrs(args);
restoreSIGPIPE();
+ /* Indicate that we managed to set up the build environment. */
+ writeFull(STDERR_FILENO, "\n");
+
/* Execute the program. This should not return. */
- inSetup = false;
- execve(program.c_str(), (char * *) &args[0], (char * *) envArr);
+ execve(drv.builder.c_str(), (char * *) &argArr[0], (char * *) &envArr[0]);
throw SysError(format("executing `%1%'") % drv.builder);
} catch (std::exception & e) {
- writeToStderr("build error: " + string(e.what()) + "\n");
- _exit(inSetup ? childSetupFailed : 1);
+ writeFull(STDERR_FILENO, "while setting up the build environment: " + string(e.what()) + "\n");
+ _exit(1);
}
abort(); /* never reached */
@@ -2333,7 +2365,7 @@ void DerivationGoal::registerOutputs()
if (buildMode == bmCheck) {
ValidPathInfo info = worker.store.queryPathInfo(path);
if (hash.first != info.hash)
- throw Error(format("derivation `%2%' may not be deterministic: hash mismatch in output `%1%'") % drvPath % path);
+ throw Error(format("derivation `%1%' may not be deterministic: hash mismatch in output `%2%'") % drvPath % path);
continue;
}
@@ -2347,16 +2379,36 @@ void DerivationGoal::registerOutputs()
debug(format("referenced input: `%1%'") % *i);
}
- /* If the derivation specifies an `allowedReferences'
- attribute (containing a list of paths that the output may
- refer to), check that all references are in that list. !!!
- allowedReferences should really be per-output. */
- if (drv.env.find("allowedReferences") != drv.env.end()) {
- PathSet allowed = parseReferenceSpecifiers(drv, get(drv.env, "allowedReferences"));
- foreach (PathSet::iterator, i, references)
- if (allowed.find(*i) == allowed.end())
- throw BuildError(format("output is not allowed to refer to path `%1%'") % *i);
- }
+ /* Enforce `allowedReferences' and friends. */
+ auto checkRefs = [&](const string & attrName, bool allowed, bool recursive) {
+ if (drv.env.find(attrName) == drv.env.end()) return;
+
+ PathSet spec = parseReferenceSpecifiers(drv, get(drv.env, attrName));
+
+ PathSet used;
+ if (recursive) {
+ /* Our requisites are the union of the closures of our references. */
+ for (auto & i : references)
+ /* Don't call computeFSClosure on ourselves. */
+ if (actualPath != i)
+ computeFSClosure(worker.store, i, used);
+ } else
+ used = references;
+
+ for (auto & i : used)
+ if (allowed) {
+ if (spec.find(i) == spec.end())
+ throw BuildError(format("output (`%1%') is not allowed to refer to path `%2%'") % actualPath % i);
+ } else {
+ if (spec.find(i) != spec.end())
+ throw BuildError(format("output (`%1%') is not allowed to refer to path `%2%'") % actualPath % i);
+ }
+ };
+
+ checkRefs("allowedReferences", true, false);
+ checkRefs("allowedRequisites", true, true);
+ checkRefs("disallowedReferences", false, false);
+ checkRefs("disallowedRequisites", false, true);
worker.store.optimisePath(path); // FIXME: combine with scanForReferences()
@@ -2475,7 +2527,7 @@ void DerivationGoal::handleChildOutput(int fd, const string & data)
BZ2_bzWrite(&err, bzLogFile, (unsigned char *) data.data(), data.size());
if (err != BZ_OK) throw Error(format("cannot write to compressed log file (BZip2 error = %1%)") % err);
} else if (fdLogFile != -1)
- writeFull(fdLogFile, (unsigned char *) data.data(), data.size());
+ writeFull(fdLogFile, data);
}
if (hook && fd == hook->fromHook.readSide)
@@ -2586,6 +2638,13 @@ public:
void cancel(bool timeout);
+ string key()
+ {
+ /* "a$" ensures substitution goals happen before derivation
+ goals. */
+ return "a$" + storePathToName(storePath) + "$" + storePath;
+ }
+
void work();
/* The states. */
@@ -2778,35 +2837,21 @@ void SubstitutionGoal::tryToRun()
args.push_back("--substitute");
args.push_back(storePath);
args.push_back(destPath);
- const char * * argArr = strings2CharPtrs(args);
+ auto argArr = stringsToCharPtrs(args);
/* Fork the substitute program. */
- pid = maybeVfork();
-
- switch (pid) {
+ pid = startProcess([&]() {
- case -1:
- throw SysError("unable to fork");
+ commonChildInit(logPipe);
- case 0:
- try { /* child */
+ if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1)
+ throw SysError("cannot dup output pipe into stdout");
- commonChildInit(logPipe);
+ execv(sub.c_str(), (char * *) &argArr[0]);
- if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1)
- throw SysError("cannot dup output pipe into stdout");
+ throw SysError(format("executing `%1%'") % sub);
+ });
- execv(sub.c_str(), (char * *) argArr);
-
- throw SysError(format("executing `%1%'") % sub);
-
- } catch (std::exception & e) {
- writeToStderr("substitute error: " + string(e.what()) + "\n");
- }
- _exit(1);
- }
-
- /* parent */
pid.setSeparatePG(true);
pid.setKillSignal(SIGTERM);
outPipe.writeSide.close();
@@ -2944,6 +2989,7 @@ Worker::Worker(LocalStore & store)
nrLocalBuilds = 0;
lastWokenUp = 0;
permanentFailure = false;
+ timedOut = false;
}
@@ -3109,15 +3155,19 @@ void Worker::run(const Goals & _topGoals)
checkInterrupt();
- /* Call every wake goal. */
+ /* Call every wake goal (in the ordering established by
+ CompareGoalPtrs). */
while (!awake.empty() && !topGoals.empty()) {
- WeakGoals awake2(awake);
+ Goals awake2;
+ for (auto & i : awake) {
+ GoalPtr goal = i.lock();
+ if (goal) awake2.insert(goal);
+ }
awake.clear();
- foreach (WeakGoals::iterator, i, awake2) {
+ for (auto & goal : awake2) {
checkInterrupt();
- GoalPtr goal = i->lock();
- if (goal) goal->work();
- if (topGoals.empty()) break;
+ goal->work();
+ if (topGoals.empty()) break; // stuff may have been cancelled
}
}
@@ -3255,6 +3305,7 @@ void Worker::waitForInput()
format("%1% timed out after %2% seconds of silence")
% goal->getName() % settings.maxSilentTime);
goal->cancel(true);
+ timedOut = true;
}
else if (goal->getExitCode() == Goal::ecBusy &&
@@ -3266,6 +3317,7 @@ void Worker::waitForInput()
format("%1% timed out after %2% seconds")
% goal->getName() % settings.buildTimeout);
goal->cancel(true);
+ timedOut = true;
}
}
@@ -3282,7 +3334,7 @@ void Worker::waitForInput()
unsigned int Worker::exitStatus()
{
- return permanentFailure ? 100 : 1;
+ return timedOut ? 101 : (permanentFailure ? 100 : 1);
}
diff --git a/nix/libstore/gc.cc b/nix/libstore/gc.cc
index f90edac1cd..34768324c2 100644
--- a/nix/libstore/gc.cc
+++ b/nix/libstore/gc.cc
@@ -96,7 +96,7 @@ Path addPermRoot(StoreAPI & store, const Path & _storePath,
"(are you running nix-build inside the store?)") % gcRoot);
if (indirect) {
- /* Don't clobber the the link if it already exists and doesn't
+ /* Don't clobber the link if it already exists and doesn't
point to the Nix store. */
if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
throw Error(format("cannot create symlink `%1%'; already exists") % gcRoot);
@@ -115,7 +115,10 @@ Path addPermRoot(StoreAPI & store, const Path & _storePath,
% gcRoot % rootsDir);
}
- makeSymlink(gcRoot, storePath);
+ if (baseNameOf(gcRoot) == baseNameOf(storePath))
+ writeFile(gcRoot, "");
+ else
+ makeSymlink(gcRoot, storePath);
}
/* Check that the root can be found by the garbage collector.
@@ -142,11 +145,6 @@ Path addPermRoot(StoreAPI & store, const Path & _storePath,
}
-/* The file to which we write our temporary roots. */
-static Path fnTempRoots;
-static AutoCloseFD fdTempRoots;
-
-
void LocalStore::addTempRoot(const Path & path)
{
/* Create the temporary roots file for this process. */
@@ -193,7 +191,7 @@ void LocalStore::addTempRoot(const Path & path)
lockFile(fdTempRoots, ltWrite, true);
string s = path + '\0';
- writeFull(fdTempRoots, (const unsigned char *) s.data(), s.size());
+ writeFull(fdTempRoots, s);
/* Downgrade to a read lock. */
debug(format("downgrading to read lock on `%1%'") % fnTempRoots);
@@ -201,27 +199,6 @@ void LocalStore::addTempRoot(const Path & path)
}
-void removeTempRoots()
-{
- if (fdTempRoots != -1) {
- fdTempRoots.close();
- unlink(fnTempRoots.c_str());
- }
-}
-
-
-/* Automatically clean up the temporary roots file when we exit. */
-struct RemoveTempRoots
-{
- ~RemoveTempRoots()
- {
- removeTempRoots();
- }
-};
-
-static RemoveTempRoots autoRemoveTempRoots __attribute__((unused));
-
-
typedef std::shared_ptr<AutoCloseFD> FDPtr;
typedef list<FDPtr> FDs;
@@ -230,11 +207,11 @@ static void readTempRoots(PathSet & tempRoots, FDs & fds)
{
/* Read the `temproots' directory for per-process temporary root
files. */
- Strings tempRootFiles = readDirectory(
+ DirEntries tempRootFiles = readDirectory(
(format("%1%/%2%") % settings.nixStateDir % tempRootsDir).str());
- foreach (Strings::iterator, i, tempRootFiles) {
- Path path = (format("%1%/%2%/%3%") % settings.nixStateDir % tempRootsDir % *i).str();
+ for (auto & i : tempRootFiles) {
+ Path path = (format("%1%/%2%/%3%") % settings.nixStateDir % tempRootsDir % i.name).str();
debug(format("reading temporary root file `%1%'") % path);
FDPtr fd(new AutoCloseFD(open(path.c_str(), O_RDWR, 0666)));
@@ -254,7 +231,7 @@ static void readTempRoots(PathSet & tempRoots, FDs & fds)
if (lockFile(*fd, ltWrite, false)) {
printMsg(lvlError, format("removing stale temporary roots file `%1%'") % path);
unlink(path.c_str());
- writeFull(*fd, (const unsigned char *) "d", 1);
+ writeFull(*fd, "d");
continue;
}
@@ -294,19 +271,19 @@ static void foundRoot(StoreAPI & store,
}
-static void findRoots(StoreAPI & store, const Path & path, Roots & roots)
+static void findRoots(StoreAPI & store, const Path & path, unsigned char type, Roots & roots)
{
try {
- struct stat st = lstat(path);
+ if (type == DT_UNKNOWN)
+ type = getFileType(path);
- if (S_ISDIR(st.st_mode)) {
- Strings names = readDirectory(path);
- foreach (Strings::iterator, i, names)
- findRoots(store, path + "/" + *i, roots);
+ if (type == DT_DIR) {
+ for (auto & i : readDirectory(path))
+ findRoots(store, path + "/" + i.name, i.type, roots);
}
- else if (S_ISLNK(st.st_mode)) {
+ else if (type == DT_LNK) {
Path target = readLink(path);
if (isInStore(target))
foundRoot(store, path, target, roots);
@@ -328,6 +305,12 @@ static void findRoots(StoreAPI & store, const Path & path, Roots & roots)
}
}
+ else if (type == DT_REG) {
+ Path storePath = settings.nixStore + "/" + baseNameOf(path);
+ if (store.isValidPath(storePath))
+ roots[path] = storePath;
+ }
+
}
catch (SysError & e) {
@@ -345,9 +328,10 @@ Roots LocalStore::findRoots()
Roots roots;
/* Process direct roots in {gcroots,manifests,profiles}. */
- nix::findRoots(*this, settings.nixStateDir + "/" + gcRootsDir, roots);
- nix::findRoots(*this, settings.nixStateDir + "/manifests", roots);
- nix::findRoots(*this, settings.nixStateDir + "/profiles", roots);
+ nix::findRoots(*this, settings.nixStateDir + "/" + gcRootsDir, DT_UNKNOWN, roots);
+ if (pathExists(settings.nixStateDir + "/manifests"))
+ nix::findRoots(*this, settings.nixStateDir + "/manifests", DT_UNKNOWN, roots);
+ nix::findRoots(*this, settings.nixStateDir + "/profiles", DT_UNKNOWN, roots);
return roots;
}
@@ -449,7 +433,6 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
// if the path was not valid, need to determine the actual
// size.
state.bytesInvalidated += size;
- // Mac OS X cannot rename directories if they are read-only.
if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
throw SysError(format("making `%1%' writable") % path);
Path tmp = state.trashDir + "/" + baseNameOf(path);
@@ -649,7 +632,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
/* After this point the set of roots or temporary roots cannot
increase, since we hold locks on everything. So everything
- that is not reachable from `roots'. */
+ that is not reachable from `roots' is garbage. */
if (state.shouldDelete) {
if (pathExists(state.trashDir)) deleteGarbage(state, state.trashDir);
@@ -741,7 +724,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
}
/* While we're at it, vacuum the database. */
- if (options.action == GCOptions::gcDeleteDead) vacuumDB();
+ //if (options.action == GCOptions::gcDeleteDead) vacuumDB();
}
diff --git a/nix/libstore/globals.cc b/nix/libstore/globals.cc
index 86fa56739c..bb08a7d0b0 100644
--- a/nix/libstore/globals.cc
+++ b/nix/libstore/globals.cc
@@ -2,6 +2,7 @@
#include "globals.hh"
#include "util.hh"
+#include "archive.hh"
#include <map>
#include <algorithm>
@@ -55,6 +56,7 @@ Settings::Settings()
envKeepDerivations = false;
lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1";
showTrace = false;
+ enableImportNative = false;
}
@@ -112,35 +114,61 @@ void Settings::set(const string & name, const string & value)
}
+string Settings::get(const string & name, const string & def)
+{
+ auto i = settings.find(name);
+ if (i == settings.end()) return def;
+ return i->second;
+}
+
+
+Strings Settings::get(const string & name, const Strings & def)
+{
+ auto i = settings.find(name);
+ if (i == settings.end()) return def;
+ return tokenizeString<Strings>(i->second);
+}
+
+
+bool Settings::get(const string & name, bool def)
+{
+ bool res = def;
+ _get(res, name);
+ return res;
+}
+
+
void Settings::update()
{
- get(tryFallback, "build-fallback");
- get(maxBuildJobs, "build-max-jobs");
- get(buildCores, "build-cores");
- get(thisSystem, "system");
- get(maxSilentTime, "build-max-silent-time");
- get(buildTimeout, "build-timeout");
- get(reservedSize, "gc-reserved-space");
- get(fsyncMetadata, "fsync-metadata");
- get(useSQLiteWAL, "use-sqlite-wal");
- get(syncBeforeRegistering, "sync-before-registering");
- get(useSubstitutes, "build-use-substitutes");
- get(buildUsersGroup, "build-users-group");
- get(useChroot, "build-use-chroot");
- get(dirsInChroot, "build-chroot-dirs");
- get(impersonateLinux26, "build-impersonate-linux-26");
- get(keepLog, "build-keep-log");
- get(compressLog, "build-compress-log");
- get(maxLogSize, "build-max-log-size");
- get(cacheFailure, "build-cache-failure");
- get(pollInterval, "build-poll-interval");
- get(checkRootReachability, "gc-check-reachability");
- get(gcKeepOutputs, "gc-keep-outputs");
- get(gcKeepDerivations, "gc-keep-derivations");
- get(autoOptimiseStore, "auto-optimise-store");
- get(envKeepDerivations, "env-keep-derivations");
- get(sshSubstituterHosts, "ssh-substituter-hosts");
- get(useSshSubstituter, "use-ssh-substituter");
+ _get(tryFallback, "build-fallback");
+ _get(maxBuildJobs, "build-max-jobs");
+ _get(buildCores, "build-cores");
+ _get(thisSystem, "system");
+ _get(maxSilentTime, "build-max-silent-time");
+ _get(buildTimeout, "build-timeout");
+ _get(reservedSize, "gc-reserved-space");
+ _get(fsyncMetadata, "fsync-metadata");
+ _get(useSQLiteWAL, "use-sqlite-wal");
+ _get(syncBeforeRegistering, "sync-before-registering");
+ _get(useSubstitutes, "build-use-substitutes");
+ _get(buildUsersGroup, "build-users-group");
+ _get(useChroot, "build-use-chroot");
+ _get(impersonateLinux26, "build-impersonate-linux-26");
+ _get(keepLog, "build-keep-log");
+ _get(compressLog, "build-compress-log");
+ _get(maxLogSize, "build-max-log-size");
+ _get(cacheFailure, "build-cache-failure");
+ _get(pollInterval, "build-poll-interval");
+ _get(checkRootReachability, "gc-check-reachability");
+ _get(gcKeepOutputs, "gc-keep-outputs");
+ _get(gcKeepDerivations, "gc-keep-derivations");
+ _get(autoOptimiseStore, "auto-optimise-store");
+ _get(envKeepDerivations, "env-keep-derivations");
+ _get(sshSubstituterHosts, "ssh-substituter-hosts");
+ _get(useSshSubstituter, "use-ssh-substituter");
+ _get(logServers, "log-servers");
+ _get(enableImportNative, "allow-unsafe-native-code-during-evaluation");
+ _get(useCaseHack, "use-case-hack");
string subs = getEnv("NIX_SUBSTITUTERS", "default");
if (subs == "default") {
@@ -158,7 +186,7 @@ void Settings::update()
}
-void Settings::get(string & res, const string & name)
+void Settings::_get(string & res, const string & name)
{
SettingsMap::iterator i = settings.find(name);
if (i == settings.end()) return;
@@ -166,7 +194,7 @@ void Settings::get(string & res, const string & name)
}
-void Settings::get(bool & res, const string & name)
+void Settings::_get(bool & res, const string & name)
{
SettingsMap::iterator i = settings.find(name);
if (i == settings.end()) return;
@@ -177,7 +205,7 @@ void Settings::get(bool & res, const string & name)
}
-void Settings::get(StringSet & res, const string & name)
+void Settings::_get(StringSet & res, const string & name)
{
SettingsMap::iterator i = settings.find(name);
if (i == settings.end()) return;
@@ -186,7 +214,7 @@ void Settings::get(StringSet & res, const string & name)
res.insert(ss.begin(), ss.end());
}
-void Settings::get(Strings & res, const string & name)
+void Settings::_get(Strings & res, const string & name)
{
SettingsMap::iterator i = settings.find(name);
if (i == settings.end()) return;
@@ -194,7 +222,7 @@ void Settings::get(Strings & res, const string & name)
}
-template<class N> void Settings::get(N & res, const string & name)
+template<class N> void Settings::_get(N & res, const string & name)
{
SettingsMap::iterator i = settings.find(name);
if (i == settings.end()) return;
diff --git a/nix/libstore/globals.hh b/nix/libstore/globals.hh
index 711c365294..c17e10d7c3 100644
--- a/nix/libstore/globals.hh
+++ b/nix/libstore/globals.hh
@@ -21,6 +21,12 @@ struct Settings {
void set(const string & name, const string & value);
+ string get(const string & name, const string & def);
+
+ Strings get(const string & name, const Strings & def);
+
+ bool get(const string & name, bool def);
+
void update();
string pack();
@@ -142,10 +148,6 @@ struct Settings {
/* Whether to build in chroot. */
bool useChroot;
- /* The directories from the host filesystem to be included in the
- chroot. */
- StringSet dirsInChroot;
-
/* Set of ssh connection strings for the ssh substituter */
Strings sshSubstituterHosts;
@@ -197,14 +199,20 @@ struct Settings {
/* Whether to show a stack trace if Nix evaluation fails. */
bool showTrace;
+ /* A list of URL prefixes that can return Nix build logs. */
+ Strings logServers;
+
+ /* Whether the importNative primop should be enabled */
+ bool enableImportNative;
+
private:
SettingsMap settings, overrides;
- void get(string & res, const string & name);
- void get(bool & res, const string & name);
- void get(StringSet & res, const string & name);
- void get(Strings & res, const string & name);
- template<class N> void get(N & res, const string & name);
+ void _get(string & res, const string & name);
+ void _get(bool & res, const string & name);
+ void _get(StringSet & res, const string & name);
+ void _get(Strings & res, const string & name);
+ template<class N> void _get(N & res, const string & name);
};
diff --git a/nix/libstore/local-store.cc b/nix/libstore/local-store.cc
index 2c3d65215c..630cb80c41 100644
--- a/nix/libstore/local-store.cc
+++ b/nix/libstore/local-store.cc
@@ -254,22 +254,25 @@ LocalStore::LocalStore(bool reserveSpace)
Path perUserDir = profilesDir + "/per-user";
createDirs(perUserDir);
if (chmod(perUserDir.c_str(), 01777) == -1)
- throw SysError(format("could not set permissions on `%1%' to 1777") % perUserDir);
+ throw SysError(format("could not set permissions on '%1%' to 1777") % perUserDir);
+
+ mode_t perm = 01775;
struct group * gr = getgrnam(settings.buildUsersGroup.c_str());
if (!gr)
throw Error(format("the group `%1%' specified in `build-users-group' does not exist")
% settings.buildUsersGroup);
-
- struct stat st;
- if (stat(settings.nixStore.c_str(), &st))
- throw SysError(format("getting attributes of path `%1%'") % settings.nixStore);
-
- if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != 01775) {
- if (chown(settings.nixStore.c_str(), 0, gr->gr_gid) == -1)
- throw SysError(format("changing ownership of path `%1%'") % settings.nixStore);
- if (chmod(settings.nixStore.c_str(), 01775) == -1)
- throw SysError(format("changing permissions on path `%1%'") % settings.nixStore);
+ else {
+ struct stat st;
+ if (stat(settings.nixStore.c_str(), &st))
+ throw SysError(format("getting attributes of path '%1%'") % settings.nixStore);
+
+ if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) {
+ if (chown(settings.nixStore.c_str(), 0, gr->gr_gid) == -1)
+ throw SysError(format("changing ownership of path '%1%'") % settings.nixStore);
+ if (chmod(settings.nixStore.c_str(), perm) == -1)
+ throw SysError(format("changing permissions on path '%1%'") % settings.nixStore);
+ }
}
}
@@ -358,7 +361,17 @@ LocalStore::~LocalStore()
i->second.to.close();
i->second.from.close();
i->second.error.close();
- i->second.pid.wait(true);
+ if (i->second.pid != -1)
+ i->second.pid.wait(true);
+ }
+ } catch (...) {
+ ignoreException();
+ }
+
+ try {
+ if (fdTempRoots != -1) {
+ fdTempRoots.close();
+ unlink(fnTempRoots.c_str());
}
} catch (...) {
ignoreException();
@@ -489,7 +502,7 @@ void LocalStore::makeStoreWritable()
if (unshare(CLONE_NEWNS) == -1)
throw SysError("setting up a private mount namespace");
- if (mount(0, settings.nixStore.c_str(), 0, MS_REMOUNT | MS_BIND, 0) == -1)
+ if (mount(0, settings.nixStore.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1)
throw SysError(format("remounting %1% writable") % settings.nixStore);
}
#endif
@@ -551,9 +564,9 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path `%1%'") % path);
- /* Really make sure that the path is of a supported type. This
- has already been checked in dumpPath(). */
- assert(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode));
+ /* Really make sure that the path is of a supported type. */
+ if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
+ throw Error(format("file ‘%1%’ has an unsupported type") % path);
/* Fail if the file is not owned by the build user. This prevents
us from messing up the ownership/permissions of files
@@ -593,9 +606,9 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
}
if (S_ISDIR(st.st_mode)) {
- Strings names = readDirectory(path);
- foreach (Strings::iterator, i, names)
- canonicalisePathMetaData_(path + "/" + *i, fromUid, inodesSeen);
+ DirEntries entries = readDirectory(path);
+ for (auto & i : entries)
+ canonicalisePathMetaData_(path + "/" + i.name, fromUid, inodesSeen);
}
}
@@ -1083,31 +1096,16 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter &
setSubstituterEnv();
- run.pid = maybeVfork();
-
- switch (run.pid) {
-
- case -1:
- throw SysError("unable to fork");
-
- case 0: /* child */
- try {
- restoreAffinity();
- if (dup2(toPipe.readSide, STDIN_FILENO) == -1)
- throw SysError("dupping stdin");
- if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1)
- throw SysError("dupping stdout");
- if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1)
- throw SysError("dupping stderr");
- execl(substituter.c_str(), substituter.c_str(), "--query", NULL);
- throw SysError(format("executing `%1%'") % substituter);
- } catch (std::exception & e) {
- std::cerr << "error: " << e.what() << std::endl;
- }
- _exit(1);
- }
-
- /* Parent. */
+ run.pid = startProcess([&]() {
+ if (dup2(toPipe.readSide, STDIN_FILENO) == -1)
+ throw SysError("dupping stdin");
+ if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1)
+ throw SysError("dupping stdout");
+ if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1)
+ throw SysError("dupping stderr");
+ execl(substituter.c_str(), substituter.c_str(), "--query", NULL);
+ throw SysError(format("executing `%1%'") % substituter);
+ });
run.program = baseNameOf(substituter);
run.to = toPipe.writeSide.borrow();
@@ -1171,7 +1169,7 @@ string LocalStore::getLineFromSubstituter(RunningSubstituter & run)
while (((p = err.find('\n')) != string::npos)
|| ((p = err.find('\r')) != string::npos)) {
string thing(err, 0, p + 1);
- writeToStderr(run.program + ": " + thing);
+ writeToStderr(run.program + ": " + thing);
err = string(err, p + 1);
}
}
@@ -1409,7 +1407,7 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
}
-Path LocalStore::addToStore(const Path & _srcPath,
+Path LocalStore::addToStore(const string & name, const Path & _srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter, bool repair)
{
Path srcPath(absPath(_srcPath));
@@ -1424,7 +1422,7 @@ Path LocalStore::addToStore(const Path & _srcPath,
else
sink.s = readFile(srcPath);
- return addToStoreFromDump(sink.s, baseNameOf(srcPath), recursive, hashAlgo, repair);
+ return addToStoreFromDump(sink.s, name, recursive, hashAlgo, repair);
}
@@ -1503,7 +1501,8 @@ void LocalStore::exportPath(const Path & path, bool sign,
{
assertStorePath(path);
- addTempRoot(path);
+ printMsg(lvlInfo, format("exporting path `%1%'") % path);
+
if (!isValidPath(path))
throw Error(format("path `%1%' is not valid") % path);
@@ -1613,8 +1612,6 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
Path dstPath = readStorePath(hashAndReadSource);
- printMsg(lvlInfo, format("importing path `%1%'") % dstPath);
-
PathSet references = readStorePaths<PathSet>(hashAndReadSource);
Path deriver = readString(hashAndReadSource);
@@ -1747,8 +1744,8 @@ bool LocalStore::verifyStore(bool checkContents, bool repair)
/* Acquire the global GC lock to prevent a garbage collection. */
AutoCloseFD fdGCLock = openGCLock(ltWrite);
- Paths entries = readDirectory(settings.nixStore);
- PathSet store(entries.begin(), entries.end());
+ PathSet store;
+ for (auto & i : readDirectory(settings.nixStore)) store.insert(i.name);
/* Check whether all valid paths actually exist. */
printMsg(lvlInfo, "checking path existence...");
@@ -1898,9 +1895,8 @@ void LocalStore::markContentsGood(const Path & path)
PathSet LocalStore::queryValidPathsOld()
{
PathSet paths;
- Strings entries = readDirectory(settings.nixDBPath + "/info");
- foreach (Strings::iterator, i, entries)
- if (i->at(0) != '.') paths.insert(settings.nixStore + "/" + *i);
+ for (auto & i : readDirectory(settings.nixDBPath + "/info"))
+ if (i.name.at(0) != '.') paths.insert(settings.nixStore + "/" + i.name);
return paths;
}
@@ -1987,9 +1983,8 @@ static void makeMutable(const Path & path)
if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) return;
if (S_ISDIR(st.st_mode)) {
- Strings names = readDirectory(path);
- foreach (Strings::iterator, i, names)
- makeMutable(path + "/" + *i);
+ for (auto & i : readDirectory(path))
+ makeMutable(path + "/" + i.name);
}
/* The O_NOFOLLOW is important to prevent us from changing the
diff --git a/nix/libstore/local-store.hh b/nix/libstore/local-store.hh
index 54331e448a..819f59327a 100644
--- a/nix/libstore/local-store.hh
+++ b/nix/libstore/local-store.hh
@@ -1,16 +1,12 @@
#pragma once
#include <string>
+#include <unordered_set>
#include "store-api.hh"
#include "util.hh"
#include "pathlocks.hh"
-#if HAVE_TR1_UNORDERED_SET
-#include <tr1/unordered_set>
-#endif
-
-
class sqlite3;
class sqlite3_stmt;
@@ -134,7 +130,7 @@ public:
void querySubstitutablePathInfos(const PathSet & paths,
SubstitutablePathInfos & infos);
- Path addToStore(const Path & srcPath,
+ Path addToStore(const string & name, const Path & srcPath,
bool recursive = true, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, bool repair = false);
@@ -171,6 +167,9 @@ public:
files with the same contents. */
void optimiseStore(OptimiseStats & stats);
+ /* Generic variant of the above method. */
+ void optimiseStore();
+
/* Optimise a single store path. */
void optimisePath(const Path & path);
@@ -245,6 +244,10 @@ private:
bool didSetSubstituterEnv;
+ /* The file to which we write our temporary roots. */
+ Path fnTempRoots;
+ AutoCloseFD fdTempRoots;
+
int getSchema();
void openDB(bool create);
@@ -306,11 +309,7 @@ private:
void checkDerivationOutputs(const Path & drvPath, const Derivation & drv);
-#if HAVE_TR1_UNORDERED_SET
- typedef std::tr1::unordered_set<ino_t> InodeHash;
-#else
- typedef std::set<ino_t> InodeHash;
-#endif
+ typedef std::unordered_set<ino_t> InodeHash;
InodeHash loadInodeHash();
Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash);
diff --git a/nix/libstore/optimise-store.cc b/nix/libstore/optimise-store.cc
index 67ee94a4bd..c62b8e451b 100644
--- a/nix/libstore/optimise-store.cc
+++ b/nix/libstore/optimise-store.cc
@@ -4,6 +4,7 @@
#include "local-store.hh"
#include "globals.hh"
+#include <cstdlib>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
@@ -225,6 +226,22 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
}
}
+static string showBytes(unsigned long long bytes)
+{
+ return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
+}
+
+void LocalStore::optimiseStore()
+{
+ OptimiseStats stats;
+
+ optimiseStore(stats);
+
+ printMsg(lvlError,
+ format("%1% freed by hard-linking %2% files")
+ % showBytes(stats.bytesFreed)
+ % stats.filesLinked);
+}
void LocalStore::optimisePath(const Path & path)
{
diff --git a/nix/libstore/pathlocks.cc b/nix/libstore/pathlocks.cc
index b858ed238d..830858ff8d 100644
--- a/nix/libstore/pathlocks.cc
+++ b/nix/libstore/pathlocks.cc
@@ -33,7 +33,7 @@ void deleteLockFile(const Path & path, int fd)
other processes waiting on this lock that the lock is stale
(deleted). */
unlink(path.c_str());
- writeFull(fd, (const unsigned char *) "d", 1);
+ writeFull(fd, "d");
/* Note that the result of unlink() is ignored; removing the lock
file is an optimisation, not a necessity. */
}
diff --git a/nix/libstore/remote-store.cc b/nix/libstore/remote-store.cc
index 4619206932..0539bbe127 100644
--- a/nix/libstore/remote-store.cc
+++ b/nix/libstore/remote-store.cc
@@ -10,6 +10,7 @@
#include <sys/stat.h>
#include <sys/socket.h>
#include <sys/un.h>
+#include <errno.h>
#include <fcntl.h>
#include <iostream>
@@ -87,8 +88,7 @@ void RemoteStore::openConnection(bool reserveSpace)
processStderr();
}
catch (Error & e) {
- throw Error(format("cannot start worker (%1%)")
- % e.msg());
+ throw Error(format("cannot start daemon worker: %1%") % e.msg());
}
setOptions();
@@ -110,7 +110,7 @@ void RemoteStore::connectToDaemon()
applications... */
AutoCloseFD fdPrevDir = open(".", O_RDONLY);
if (fdPrevDir == -1) throw SysError("couldn't open current directory");
- chdir(dirOf(socketPath).c_str());
+ if (chdir(dirOf(socketPath).c_str()) == -1) throw SysError(format("couldn't change to directory of ‘%1%’") % socketPath);
Path socketPathRel = "./" + baseNameOf(socketPath);
struct sockaddr_un addr;
@@ -133,8 +133,6 @@ RemoteStore::~RemoteStore()
try {
to.flush();
fdSocket.close();
- if (child != -1)
- child.wait(true);
} catch (...) {
ignoreException();
}
@@ -387,7 +385,7 @@ Path RemoteStore::queryPathFromHashPart(const string & hashPart)
}
-Path RemoteStore::addToStore(const Path & _srcPath,
+Path RemoteStore::addToStore(const string & name, const Path & _srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter, bool repair)
{
if (repair) throw Error("repairing is not supported when building through the Nix daemon");
@@ -397,13 +395,28 @@ Path RemoteStore::addToStore(const Path & _srcPath,
Path srcPath(absPath(_srcPath));
writeInt(wopAddToStore, to);
- writeString(baseNameOf(srcPath), to);
+ writeString(name, to);
/* backwards compatibility hack */
writeInt((hashAlgo == htSHA256 && recursive) ? 0 : 1, to);
writeInt(recursive ? 1 : 0, to);
writeString(printHashType(hashAlgo), to);
- dumpPath(srcPath, to, filter);
- processStderr();
+
+ try {
+ to.written = 0;
+ to.warn = true;
+ dumpPath(srcPath, to, filter);
+ to.warn = false;
+ processStderr();
+ } catch (SysError & e) {
+ /* Daemon closed while we were sending the path. Probably OOM
+ or I/O error. */
+ if (e.errNo == EPIPE)
+ try {
+ processStderr();
+ } catch (EndOfFile & e) { }
+ throw;
+ }
+
return readStorePath(from);
}
@@ -564,6 +577,23 @@ void RemoteStore::clearFailedPaths(const PathSet & paths)
readInt(from);
}
+void RemoteStore::optimiseStore()
+{
+ openConnection();
+ writeInt(wopOptimiseStore, to);
+ processStderr();
+ readInt(from);
+}
+
+bool RemoteStore::verifyStore(bool checkContents, bool repair)
+{
+ openConnection();
+ writeInt(wopVerifyStore, to);
+ writeInt(checkContents, to);
+ writeInt(repair, to);
+ processStderr();
+ return readInt(from) != 0;
+}
void RemoteStore::processStderr(Sink * sink, Source * source)
{
diff --git a/nix/libstore/remote-store.hh b/nix/libstore/remote-store.hh
index 04b60fce4b..030120db40 100644
--- a/nix/libstore/remote-store.hh
+++ b/nix/libstore/remote-store.hh
@@ -21,15 +21,15 @@ public:
RemoteStore();
~RemoteStore();
-
+
/* Implementations of abstract store API methods. */
-
+
bool isValidPath(const Path & path);
PathSet queryValidPaths(const PathSet & paths);
-
+
PathSet queryAllValidPaths();
-
+
ValidPathInfo queryPathInfo(const Path & path);
Hash queryPathHash(const Path & path);
@@ -39,21 +39,21 @@ public:
void queryReferrers(const Path & path, PathSet & referrers);
Path queryDeriver(const Path & path);
-
+
PathSet queryValidDerivers(const Path & path);
PathSet queryDerivationOutputs(const Path & path);
-
+
StringSet queryDerivationOutputNames(const Path & path);
Path queryPathFromHashPart(const string & hashPart);
-
+
PathSet querySubstitutablePaths(const PathSet & paths);
-
+
void querySubstitutablePathInfos(const PathSet & paths,
SubstitutablePathInfos & infos);
-
- Path addToStore(const Path & srcPath,
+
+ Path addToStore(const string & name, const Path & srcPath,
bool recursive = true, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, bool repair = false);
@@ -64,7 +64,7 @@ public:
Sink & sink);
Paths importPaths(bool requireSignature, Source & source);
-
+
void buildPaths(const PathSet & paths, BuildMode buildMode);
void ensurePath(const Path & path);
@@ -72,22 +72,24 @@ public:
void addTempRoot(const Path & path);
void addIndirectRoot(const Path & path);
-
+
void syncWithGC();
-
+
Roots findRoots();
void collectGarbage(const GCOptions & options, GCResults & results);
-
+
PathSet queryFailedPaths();
void clearFailedPaths(const PathSet & paths);
-
+
+ void optimiseStore();
+
+ bool verifyStore(bool checkContents, bool repair);
private:
AutoCloseFD fdSocket;
FdSink to;
FdSource from;
- Pid child;
unsigned int daemonVersion;
bool initialised;
diff --git a/nix/libstore/store-api.hh b/nix/libstore/store-api.hh
index b635fee2cf..3764f3e542 100644
--- a/nix/libstore/store-api.hh
+++ b/nix/libstore/store-api.hh
@@ -54,7 +54,7 @@ struct GCOptions
};
-struct GCResults
+struct GCResults
{
/* Depending on the action, the GC roots, or the paths that would
be or have been deleted. */
@@ -82,7 +82,7 @@ struct SubstitutablePathInfo
typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos;
-struct ValidPathInfo
+struct ValidPathInfo
{
Path path;
Path deriver;
@@ -100,13 +100,13 @@ typedef list<ValidPathInfo> ValidPathInfos;
enum BuildMode { bmNormal, bmRepair, bmCheck };
-class StoreAPI
+class StoreAPI
{
public:
virtual ~StoreAPI() { }
- /* Check whether a path is valid. */
+ /* Check whether a path is valid. */
virtual bool isValidPath(const Path & path) = 0;
/* Query which of the given paths is valid. */
@@ -118,7 +118,7 @@ public:
/* Query information about a valid path. */
virtual ValidPathInfo queryPathInfo(const Path & path) = 0;
- /* Query the hash of a valid path. */
+ /* Query the hash of a valid path. */
virtual Hash queryPathHash(const Path & path) = 0;
/* Query the set of outgoing FS references for a store path. The
@@ -150,7 +150,7 @@ public:
/* Query the full store path given the hash part of a valid store
path, or "" if the path doesn't exist. */
virtual Path queryPathFromHashPart(const string & hashPart) = 0;
-
+
/* Query which of the given paths have substitutes. */
virtual PathSet querySubstitutablePaths(const PathSet & paths) = 0;
@@ -159,12 +159,12 @@ public:
info, it's omitted from the resulting ‘infos’ map. */
virtual void querySubstitutablePathInfos(const PathSet & paths,
SubstitutablePathInfos & infos) = 0;
-
+
/* Copy the contents of a path to the store and register the
validity the resulting path. The resulting path is returned.
The function object `filter' can be used to exclude files (see
libutil/archive.hh). */
- virtual Path addToStore(const Path & srcPath,
+ virtual Path addToStore(const string & name, const Path & srcPath,
bool recursive = true, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, bool repair = false) = 0;
@@ -250,6 +250,14 @@ public:
`nix-store --register-validity'. */
string makeValidityRegistration(const PathSet & paths,
bool showDerivers, bool showHash);
+
+ /* Optimise the disk space usage of the Nix store by hard-linking files
+ with the same contents. */
+ virtual void optimiseStore() = 0;
+
+ /* Check the integrity of the Nix store. Returns true if errors
+ remain. */
+ virtual bool verifyStore(bool checkContents, bool repair) = 0;
};
@@ -263,7 +271,7 @@ bool isStorePath(const Path & path);
/* Extract the name part of the given store path. */
string storePathToName(const Path & path);
-
+
void checkStoreName(const string & name);
@@ -284,7 +292,7 @@ Path followLinksToStorePath(const Path & path);
/* Constructs a unique store path name. */
Path makeStorePath(const string & type,
const Hash & hash, const string & name);
-
+
Path makeOutputPath(const string & id,
const Hash & hash, const string & name);
diff --git a/nix/libstore/worker-protocol.hh b/nix/libstore/worker-protocol.hh
index 9317f89c37..d037d7402e 100644
--- a/nix/libstore/worker-protocol.hh
+++ b/nix/libstore/worker-protocol.hh
@@ -12,7 +12,6 @@ namespace nix {
typedef enum {
- wopQuit = 0,
wopIsValidPath = 1,
wopHasSubstitutes = 3,
wopQueryPathHash = 4,
@@ -43,6 +42,8 @@ typedef enum {
wopQueryValidPaths = 31,
wopQuerySubstitutablePaths = 32,
wopQueryValidDerivers = 33,
+ wopOptimiseStore = 34,
+ wopVerifyStore = 35
} WorkerOp;
diff --git a/nix/libutil/archive.cc b/nix/libutil/archive.cc
index 70a1c580dd..6856ea0f28 100644
--- a/nix/libutil/archive.cc
+++ b/nix/libutil/archive.cc
@@ -1,10 +1,14 @@
+#define _XOPEN_SOURCE 600
+
#include "config.h"
#include <cerrno>
#include <algorithm>
#include <vector>
+#include <map>
+
+#include <strings.h> // for strcasecmp
-#define _XOPEN_SOURCE 600
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
@@ -18,39 +22,21 @@
namespace nix {
+bool useCaseHack =
+#if __APPLE__
+ true;
+#else
+ false;
+#endif
+
static string archiveVersion1 = "nix-archive-1";
+static string caseHackSuffix = "~nix~case~hack~";
PathFilter defaultPathFilter;
-static void dump(const string & path, Sink & sink, PathFilter & filter);
-
-
-static void dumpEntries(const Path & path, Sink & sink, PathFilter & filter)
-{
- Strings names = readDirectory(path);
- vector<string> names2(names.begin(), names.end());
- sort(names2.begin(), names2.end());
-
- for (vector<string>::iterator i = names2.begin();
- i != names2.end(); ++i)
- {
- Path entry = path + "/" + *i;
- if (filter(entry)) {
- writeString("entry", sink);
- writeString("(", sink);
- writeString("name", sink);
- writeString(*i, sink);
- writeString("node", sink);
- dump(entry, sink, filter);
- writeString(")", sink);
- }
- }
-}
-
-
-static void dumpContents(const Path & path, size_t size,
+static void dumpContents(const Path & path, size_t size,
Sink & sink)
{
writeString("contents", sink);
@@ -58,7 +44,7 @@ static void dumpContents(const Path & path, size_t size,
AutoCloseFD fd = open(path.c_str(), O_RDONLY);
if (fd == -1) throw SysError(format("opening file `%1%'") % path);
-
+
unsigned char buf[65536];
size_t left = size;
@@ -89,12 +75,40 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
writeString("", sink);
}
dumpContents(path, (size_t) st.st_size, sink);
- }
+ }
else if (S_ISDIR(st.st_mode)) {
writeString("type", sink);
writeString("directory", sink);
- dumpEntries(path, sink, filter);
+
+ /* If we're on a case-insensitive system like Mac OS X, undo
+ the case hack applied by restorePath(). */
+ std::map<string, string> unhacked;
+ for (auto & i : readDirectory(path))
+ if (useCaseHack) {
+ string name(i.name);
+ size_t pos = i.name.find(caseHackSuffix);
+ if (pos != string::npos) {
+ printMsg(lvlDebug, format("removing case hack suffix from `%1%'") % (path + "/" + i.name));
+ name.erase(pos);
+ }
+ if (unhacked.find(name) != unhacked.end())
+ throw Error(format("file name collision in between `%1%' and `%2%'")
+ % (path + "/" + unhacked[name]) % (path + "/" + i.name));
+ unhacked[name] = i.name;
+ } else
+ unhacked[i.name] = i.name;
+
+ for (auto & i : unhacked)
+ if (filter(path + "/" + i.first)) {
+ writeString("entry", sink);
+ writeString("(", sink);
+ writeString("name", sink);
+ writeString(i.first, sink);
+ writeString("node", sink);
+ dump(path + "/" + i.second, sink, filter);
+ writeString(")", sink);
+ }
}
else if (S_ISLNK(st.st_mode)) {
@@ -123,6 +137,7 @@ static SerialisationError badArchive(string s)
}
+#if 0
static void skipGeneric(Source & source)
{
if (readString(source) == "(") {
@@ -130,43 +145,13 @@ static void skipGeneric(Source & source)
skipGeneric(source);
}
}
-
-
-static void parse(ParseSink & sink, Source & source, const Path & path);
-
-
-
-static void parseEntry(ParseSink & sink, Source & source, const Path & path)
-{
- string s, name;
-
- s = readString(source);
- if (s != "(") throw badArchive("expected open tag");
-
- while (1) {
- checkInterrupt();
-
- s = readString(source);
-
- if (s == ")") {
- break;
- } else if (s == "name") {
- name = readString(source);
- } else if (s == "node") {
- if (s == "") throw badArchive("entry name missing");
- parse(sink, source, path + "/" + name);
- } else {
- throw badArchive("unknown field " + s);
- skipGeneric(source);
- }
- }
-}
+#endif
static void parseContents(ParseSink & sink, Source & source, const Path & path)
{
unsigned long long size = readLongLong(source);
-
+
sink.preallocateContents(size);
unsigned long long left = size;
@@ -185,6 +170,15 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path)
}
+struct CaseInsensitiveCompare
+{
+ bool operator() (const string & a, const string & b) const
+ {
+ return strcasecmp(a.c_str(), b.c_str()) < 0;
+ }
+};
+
+
static void parse(ParseSink & sink, Source & source, const Path & path)
{
string s;
@@ -194,6 +188,8 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown;
+ std::map<Path, int, CaseInsensitiveCompare> names;
+
while (1) {
checkInterrupt();
@@ -221,9 +217,9 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
else if (t == "symlink") {
type = tpSymlink;
}
-
+
else throw badArchive("unknown file type " + t);
-
+
}
else if (s == "contents" && type == tpRegular) {
@@ -236,7 +232,40 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
}
else if (s == "entry" && type == tpDirectory) {
- parseEntry(sink, source, path);
+ string name, prevName;
+
+ s = readString(source);
+ if (s != "(") throw badArchive("expected open tag");
+
+ while (1) {
+ checkInterrupt();
+
+ s = readString(source);
+
+ if (s == ")") {
+ break;
+ } else if (s == "name") {
+ name = readString(source);
+ if (name.empty() || name == "." || name == ".." || name.find('/') != string::npos || name.find((char) 0) != string::npos)
+ throw Error(format("NAR contains invalid file name `%1%'") % name);
+ if (name <= prevName)
+ throw Error("NAR directory is not sorted");
+ prevName = name;
+ if (useCaseHack) {
+ auto i = names.find(name);
+ if (i != names.end()) {
+ printMsg(lvlDebug, format("case collision between `%1%' and `%2%'") % i->first % name);
+ name += caseHackSuffix;
+ name += int2String(++i->second);
+ } else
+ names[name] = 0;
+ }
+ } else if (s == "node") {
+ if (s.empty()) throw badArchive("entry name missing");
+ parse(sink, source, path + "/" + name);
+ } else
+ throw badArchive("unknown field " + s);
+ }
}
else if (s == "target" && type == tpSymlink) {
@@ -244,17 +273,15 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
sink.createSymlink(path, target);
}
- else {
+ else
throw badArchive("unknown field " + s);
- skipGeneric(source);
- }
}
}
void parseDump(ParseSink & sink, Source & source)
{
- string version;
+ string version;
try {
version = readString(source);
} catch (SerialisationError & e) {
@@ -323,7 +350,7 @@ struct RestoreSink : ParseSink
}
};
-
+
void restorePath(const Path & path, Source & source)
{
RestoreSink sink;
@@ -331,5 +358,5 @@ void restorePath(const Path & path, Source & source)
parseDump(sink, source);
}
-
+
}
diff --git a/nix/libutil/archive.hh b/nix/libutil/archive.hh
index ccac92074d..c216e9768f 100644
--- a/nix/libutil/archive.hh
+++ b/nix/libutil/archive.hh
@@ -28,7 +28,7 @@ namespace nix {
where:
- attrs(as) = concat(map(attr, as)) + encN(0)
+ attrs(as) = concat(map(attr, as)) + encN(0)
attrs((a, b)) = encS(a) + encS(b)
encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary)
@@ -58,7 +58,7 @@ void dumpPath(const Path & path, Sink & sink,
struct ParseSink
{
virtual void createDirectory(const Path & path) { };
-
+
virtual void createRegularFile(const Path & path) { };
virtual void isExecutable() { };
virtual void preallocateContents(unsigned long long size) { };
@@ -66,10 +66,14 @@ struct ParseSink
virtual void createSymlink(const Path & path, const string & target) { };
};
-
+
void parseDump(ParseSink & sink, Source & source);
void restorePath(const Path & path, Source & source);
-
+
+// FIXME: global variables are bad m'kay.
+extern bool useCaseHack;
+
+
}
diff --git a/nix/libutil/hash.cc b/nix/libutil/hash.cc
index 050446610f..2da00a53de 100644
--- a/nix/libutil/hash.cc
+++ b/nix/libutil/hash.cc
@@ -84,7 +84,7 @@ string printHash(const Hash & hash)
return string(buf, hash.hashSize * 2);
}
-
+
Hash parseHash(HashType ht, const string & s)
{
Hash hash(ht);
@@ -92,7 +92,7 @@ Hash parseHash(HashType ht, const string & s)
throw Error(format("invalid hash `%1%'") % s);
for (unsigned int i = 0; i < hash.hashSize; i++) {
string s2(s, i * 2, 2);
- if (!isxdigit(s2[0]) || !isxdigit(s2[1]))
+ if (!isxdigit(s2[0]) || !isxdigit(s2[1]))
throw Error(format("invalid hash `%1%'") % s);
std::istringstream str(s2);
int n;
@@ -103,24 +103,6 @@ Hash parseHash(HashType ht, const string & s)
}
-static unsigned char divMod(unsigned char * bytes, unsigned char y)
-{
- unsigned int borrow = 0;
-
- int pos = Hash::maxHashSize - 1;
- while (pos >= 0 && !bytes[pos]) --pos;
-
- for ( ; pos >= 0; --pos) {
- unsigned int s = bytes[pos] + (borrow << 8);
- unsigned int d = s / y;
- borrow = s % y;
- bytes[pos] = d;
- }
-
- return borrow;
-}
-
-
unsigned int hashLength32(const Hash & hash)
{
return (hash.hashSize * 8 - 1) / 5 + 1;
@@ -136,19 +118,19 @@ string printHash32(const Hash & hash)
Hash hash2(hash);
unsigned int len = hashLength32(hash);
- const char * chars = base32Chars.data();
-
- string s(len, '0');
-
- int pos = len - 1;
- while (pos >= 0) {
- unsigned char digit = divMod(hash2.hash, 32);
- s[pos--] = chars[digit];
+ string s;
+ s.reserve(len);
+
+ for (int n = len - 1; n >= 0; n--) {
+ unsigned int b = n * 5;
+ unsigned int i = b / 8;
+ unsigned int j = b % 8;
+ unsigned char c =
+ (hash.hash[i] >> j)
+ | (i >= hash.hashSize - 1 ? 0 : hash.hash[i + 1] << (8 - j));
+ s.push_back(base32Chars[c & 0x1f]);
}
- for (unsigned int i = 0; i < hash2.maxHashSize; ++i)
- assert(hash2.hash[i] == 0);
-
return s;
}
@@ -159,51 +141,24 @@ string printHash16or32(const Hash & hash)
}
-static bool mul(unsigned char * bytes, unsigned char y, int maxSize)
-{
- unsigned char carry = 0;
-
- for (int pos = 0; pos < maxSize; ++pos) {
- unsigned int m = bytes[pos] * y + carry;
- bytes[pos] = m & 0xff;
- carry = m >> 8;
- }
-
- return carry;
-}
-
-
-static bool add(unsigned char * bytes, unsigned char y, int maxSize)
-{
- unsigned char carry = y;
-
- for (int pos = 0; pos < maxSize; ++pos) {
- unsigned int m = bytes[pos] + carry;
- bytes[pos] = m & 0xff;
- carry = m >> 8;
- if (carry == 0) break;
- }
-
- return carry;
-}
-
-
Hash parseHash32(HashType ht, const string & s)
{
Hash hash(ht);
+ unsigned int len = hashLength32(ht);
+ assert(s.size() == len);
- const char * chars = base32Chars.data();
-
- for (unsigned int i = 0; i < s.length(); ++i) {
- char c = s[i];
+ for (unsigned int n = 0; n < len; ++n) {
+ char c = s[len - n - 1];
unsigned char digit;
for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */
- if (chars[digit] == c) break;
+ if (base32Chars[digit] == c) break;
if (digit >= 32)
- throw Error(format("invalid base-32 hash `%1%'") % s);
- if (mul(hash.hash, 32, hash.hashSize) ||
- add(hash.hash, digit, hash.hashSize))
- throw Error(format("base-32 hash `%1%' is too large") % s);
+ throw Error(format("invalid base-32 hash '%1%'") % s);
+ unsigned int b = n * 5;
+ unsigned int i = b / 8;
+ unsigned int j = b % 8;
+ hash.hash[i] |= digit << j;
+ if (i < hash.hashSize - 1) hash.hash[i + 1] |= digit >> (8 - j);
}
return hash;
@@ -299,7 +254,7 @@ Hash hashFile(HashType ht, const Path & path)
if (n == -1) throw SysError(format("reading file `%1%'") % path);
update(ht, ctx, buf, n);
}
-
+
finish(ht, ctx, hash.hash);
return hash;
}
@@ -311,7 +266,7 @@ HashSink::HashSink(HashType ht) : ht(ht)
bytes = 0;
start(ht, *ctx);
}
-
+
HashSink::~HashSink()
{
bufPos = 0;
@@ -369,7 +324,7 @@ HashType parseHashType(const string & s)
else return htUnknown;
}
-
+
string printHashType(HashType ht)
{
if (ht == htMD5) return "md5";
@@ -378,5 +333,5 @@ string printHashType(HashType ht)
else throw Error("cannot print unknown hash type");
}
-
+
}
diff --git a/nix/libutil/serialise.cc b/nix/libutil/serialise.cc
index 6b71f52c15..9241750750 100644
--- a/nix/libutil/serialise.cc
+++ b/nix/libutil/serialise.cc
@@ -54,8 +54,24 @@ FdSink::~FdSink()
}
+size_t threshold = 256 * 1024 * 1024;
+
+static void warnLargeDump()
+{
+ printMsg(lvlError, "warning: dumping very large path (> 256 MiB); this may run out of memory");
+}
+
+
void FdSink::write(const unsigned char * data, size_t len)
{
+ static bool warned = false;
+ if (warn && !warned) {
+ written += len;
+ if (written > threshold) {
+ warnLargeDump();
+ warned = true;
+ }
+ }
writeFull(fd, data, len);
}
@@ -256,4 +272,15 @@ template Paths readStrings(Source & source);
template PathSet readStrings(Source & source);
+void StringSink::operator () (const unsigned char * data, size_t len)
+{
+ static bool warned = false;
+ if (!warned && s.size() > threshold) {
+ warnLargeDump();
+ warned = true;
+ }
+ s.append((const char *) data, len);
+}
+
+
}
diff --git a/nix/libutil/serialise.hh b/nix/libutil/serialise.hh
index e5a9df1d05..6a6f028aa6 100644
--- a/nix/libutil/serialise.hh
+++ b/nix/libutil/serialise.hh
@@ -72,9 +72,11 @@ struct BufferedSource : Source
struct FdSink : BufferedSink
{
int fd;
+ bool warn;
+ size_t written;
- FdSink() : fd(-1) { }
- FdSink(int fd) : fd(fd) { }
+ FdSink() : fd(-1), warn(false), written(0) { }
+ FdSink(int fd) : fd(fd), warn(false), written(0) { }
~FdSink();
void write(const unsigned char * data, size_t len);
@@ -95,10 +97,7 @@ struct FdSource : BufferedSource
struct StringSink : Sink
{
string s;
- void operator () (const unsigned char * data, size_t len)
- {
- s.append((const char *) data, len);
- }
+ void operator () (const unsigned char * data, size_t len);
};
diff --git a/nix/libutil/types.hh b/nix/libutil/types.hh
index 4b5ce9a78c..160884ee1a 100644
--- a/nix/libutil/types.hh
+++ b/nix/libutil/types.hh
@@ -8,6 +8,15 @@
#include <boost/format.hpp>
+/* Before 4.7, gcc's std::exception uses empty throw() specifiers for
+ * its (virtual) destructor and what() in c++11 mode, in violation of spec
+ */
+#ifdef __GNUC__
+#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7)
+#define EXCEPTION_NEEDS_THROW_SPEC
+#endif
+#endif
+
namespace nix {
@@ -39,10 +48,14 @@ protected:
public:
unsigned int status; // exit status
BaseError(const FormatOrString & fs, unsigned int status = 1);
+#ifdef EXCEPTION_NEEDS_THROW_SPEC
~BaseError() throw () { };
const char * what() const throw () { return err.c_str(); }
- const string & msg() const throw () { return err; }
- const string & prefix() const throw () { return prefix_; }
+#else
+ const char * what() const noexcept { return err.c_str(); }
+#endif
+ const string & msg() const { return err; }
+ const string & prefix() const { return prefix_; }
BaseError & addPrefix(const FormatOrString & fs);
};
diff --git a/nix/libutil/util.cc b/nix/libutil/util.cc
index 846674a29d..dab4235b04 100644
--- a/nix/libutil/util.cc
+++ b/nix/libutil/util.cc
@@ -1,5 +1,8 @@
#include "config.h"
+#include "util.hh"
+#include "affinity.hh"
+
#include <iostream>
#include <cerrno>
#include <cstdio>
@@ -16,7 +19,9 @@
#include <sys/syscall.h>
#endif
-#include "util.hh"
+#ifdef __linux__
+#include <sys/prctl.h>
+#endif
extern char * * environ;
@@ -125,7 +130,6 @@ Path canonPath(const Path & path, bool resolveSymlinks)
i = temp.begin(); /* restart */
end = temp.end();
s = "";
- /* !!! potential for infinite loop */
}
}
}
@@ -189,8 +193,12 @@ Path readLink(const Path & path)
if (!S_ISLNK(st.st_mode))
throw Error(format("`%1%' is not a symlink") % path);
char buf[st.st_size];
- if (readlink(path.c_str(), buf, st.st_size) != st.st_size)
- throw SysError(format("reading symbolic link `%1%'") % path);
+ ssize_t rlsize = readlink(path.c_str(), buf, st.st_size);
+ if (rlsize == -1)
+ throw SysError(format("reading symbolic link '%1%'") % path);
+ else if (rlsize > st.st_size)
+ throw Error(format("symbolic link ‘%1%’ size overflow %2% > %3%")
+ % path % rlsize % st.st_size);
return string(buf, st.st_size);
}
@@ -202,9 +210,10 @@ bool isLink(const Path & path)
}
-Strings readDirectory(const Path & path)
+DirEntries readDirectory(const Path & path)
{
- Strings names;
+ DirEntries entries;
+ entries.reserve(64);
AutoCloseDir dir = opendir(path.c_str());
if (!dir) throw SysError(format("opening directory `%1%'") % path);
@@ -214,11 +223,21 @@ Strings readDirectory(const Path & path)
checkInterrupt();
string name = dirent->d_name;
if (name == "." || name == "..") continue;
- names.push_back(name);
+ entries.emplace_back(name, dirent->d_ino, dirent->d_type);
}
if (errno) throw SysError(format("reading directory `%1%'") % path);
- return names;
+ return entries;
+}
+
+
+unsigned char getFileType(const Path & path)
+{
+ struct stat st = lstat(path);
+ if (S_ISDIR(st.st_mode)) return DT_DIR;
+ if (S_ISLNK(st.st_mode)) return DT_LNK;
+ if (S_ISREG(st.st_mode)) return DT_REG;
+ return DT_UNKNOWN;
}
@@ -249,8 +268,8 @@ void writeFile(const Path & path, const string & s)
{
AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0666);
if (fd == -1)
- throw SysError(format("opening file `%1%'") % path);
- writeFull(fd, (unsigned char *) s.data(), s.size());
+ throw SysError(format("opening file '%1%'") % path);
+ writeFull(fd, s);
}
@@ -277,7 +296,7 @@ string readLine(int fd)
void writeLine(int fd, string s)
{
s += '\n';
- writeFull(fd, (const unsigned char *) s.data(), s.size());
+ writeFull(fd, s);
}
@@ -293,16 +312,14 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
bytesFreed += st.st_blocks * 512;
if (S_ISDIR(st.st_mode)) {
- Strings names = readDirectory(path);
-
/* Make the directory writable. */
if (!(st.st_mode & S_IWUSR)) {
if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
throw SysError(format("making `%1%' writable") % path);
}
- for (Strings::iterator i = names.begin(); i != names.end(); ++i)
- _deletePath(path + "/" + *i, bytesFreed);
+ for (auto & i : readDirectory(path))
+ _deletePath(path + "/" + i.name, bytesFreed);
}
if (remove(path.c_str()) == -1)
@@ -380,6 +397,9 @@ Paths createDirs(const Path & path)
created.push_back(path);
}
+ if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1)
+ throw SysError(format("statting symlink `%1%'") % path);
+
if (!S_ISDIR(st.st_mode)) throw Error(format("`%1%' is not a directory") % path);
return created;
@@ -469,7 +489,10 @@ void warnOnce(bool & haveWarned, const FormatOrString & fs)
void writeToStderr(const string & s)
{
try {
- _writeToStderr((const unsigned char *) s.data(), s.size());
+ if (_writeToStderr)
+ _writeToStderr((const unsigned char *) s.data(), s.size());
+ else
+ writeFull(STDERR_FILENO, s);
} catch (SysError & e) {
/* Ignore failing writes to stderr if we're in an exception
handler, otherwise throw an exception. We need to ignore
@@ -481,13 +504,7 @@ void writeToStderr(const string & s)
}
-static void defaultWriteToStderr(const unsigned char * buf, size_t count)
-{
- writeFull(STDERR_FILENO, buf, count);
-}
-
-
-void (*_writeToStderr) (const unsigned char * buf, size_t count) = defaultWriteToStderr;
+void (*_writeToStderr) (const unsigned char * buf, size_t count) = 0;
void readFull(int fd, unsigned char * buf, size_t count)
@@ -521,6 +538,12 @@ void writeFull(int fd, const unsigned char * buf, size_t count)
}
+void writeFull(int fd, const string & s)
+{
+ writeFull(fd, (const unsigned char *) s.data(), s.size());
+}
+
+
string drainFD(int fd)
{
string result;
@@ -707,10 +730,14 @@ void AutoCloseDir::close()
Pid::Pid()
+ : pid(-1), separatePG(false), killSignal(SIGKILL)
+{
+}
+
+
+Pid::Pid(pid_t pid)
+ : pid(pid), separatePG(false), killSignal(SIGKILL)
{
- pid = -1;
- separatePG = false;
- killSignal = SIGKILL;
}
@@ -734,11 +761,12 @@ Pid::operator pid_t()
}
-void Pid::kill()
+void Pid::kill(bool quiet)
{
if (pid == -1 || pid == 0) return;
- printMsg(lvlError, format("killing process %1%") % pid);
+ if (!quiet)
+ printMsg(lvlError, format("killing process %1%") % pid);
/* Send the requested signal to the child. If it has its own
process group, send the signal to every process in the child
@@ -801,43 +829,30 @@ void killUser(uid_t uid)
users to which the current process can send signals. So we
fork a process, switch to uid, and send a mass kill. */
- Pid pid;
- pid = fork();
- switch (pid) {
-
- case -1:
- throw SysError("unable to fork");
-
- case 0:
- try { /* child */
+ Pid pid = startProcess([&]() {
- if (setuid(uid) == -1)
- throw SysError("setting uid");
+ if (setuid(uid) == -1)
+ throw SysError("setting uid");
- while (true) {
+ while (true) {
#ifdef __APPLE__
- /* OSX's kill syscall takes a third parameter that, among other
- things, determines if kill(-1, signo) affects the calling
- process. In the OSX libc, it's set to true, which means
- "follow POSIX", which we don't want here
+ /* OSX's kill syscall takes a third parameter that, among
+ other things, determines if kill(-1, signo) affects the
+ calling process. In the OSX libc, it's set to true,
+ which means "follow POSIX", which we don't want here
*/
- if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break;
+ if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break;
#else
- if (kill(-1, SIGKILL) == 0) break;
+ if (kill(-1, SIGKILL) == 0) break;
#endif
- if (errno == ESRCH) break; /* no more processes */
- if (errno != EINTR)
- throw SysError(format("cannot kill processes for uid `%1%'") % uid);
- }
-
- } catch (std::exception & e) {
- writeToStderr((format("killing processes belonging to uid `%1%': %2%\n") % uid % e.what()).str());
- _exit(1);
+ if (errno == ESRCH) break; /* no more processes */
+ if (errno != EINTR)
+ throw SysError(format("cannot kill processes for uid `%1%'") % uid);
}
+
_exit(0);
- }
+ });
- /* parent */
int status = pid.wait(true);
if (status != 0)
throw Error(format("cannot kill processes for uid `%1%': %2%") % uid % statusToString(status));
@@ -852,47 +867,69 @@ void killUser(uid_t uid)
//////////////////////////////////////////////////////////////////////
+pid_t startProcess(std::function<void()> fun,
+ bool dieWithParent, const string & errorPrefix, bool runExitHandlers)
+{
+ pid_t pid = fork();
+ if (pid == -1) throw SysError("unable to fork");
+
+ if (pid == 0) {
+ _writeToStderr = 0;
+ try {
+#if __linux__
+ if (dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1)
+ throw SysError("setting death signal");
+#endif
+ restoreAffinity();
+ fun();
+ } catch (std::exception & e) {
+ try {
+ std::cerr << errorPrefix << e.what() << "\n";
+ } catch (...) { }
+ } catch (...) { }
+ if (runExitHandlers)
+ exit(1);
+ else
+ _exit(1);
+ }
+
+ return pid;
+}
+
+
+std::vector<const char *> stringsToCharPtrs(const Strings & ss)
+{
+ std::vector<const char *> res;
+ for (auto & s : ss) res.push_back(s.c_str());
+ res.push_back(0);
+ return res;
+}
+
+
string runProgram(Path program, bool searchPath, const Strings & args)
{
checkInterrupt();
- std::vector<const char *> cargs; /* careful with c_str()! */
- cargs.push_back(program.c_str());
- for (Strings::const_iterator i = args.begin(); i != args.end(); ++i)
- cargs.push_back(i->c_str());
- cargs.push_back(0);
-
/* Create a pipe. */
Pipe pipe;
pipe.create();
/* Fork. */
- Pid pid;
- pid = maybeVfork();
-
- switch (pid) {
-
- case -1:
- throw SysError("unable to fork");
-
- case 0: /* child */
- try {
- if (dup2(pipe.writeSide, STDOUT_FILENO) == -1)
- throw SysError("dupping stdout");
+ Pid pid = startProcess([&]() {
+ if (dup2(pipe.writeSide, STDOUT_FILENO) == -1)
+ throw SysError("dupping stdout");
- if (searchPath)
- execvp(program.c_str(), (char * *) &cargs[0]);
- else
- execv(program.c_str(), (char * *) &cargs[0]);
- throw SysError(format("executing `%1%'") % program);
+ Strings args_(args);
+ args_.push_front(program);
+ auto cargs = stringsToCharPtrs(args_);
- } catch (std::exception & e) {
- writeToStderr("error: " + string(e.what()) + "\n");
- }
- _exit(1);
- }
+ if (searchPath)
+ execvp(program.c_str(), (char * *) &cargs[0]);
+ else
+ execv(program.c_str(), (char * *) &cargs[0]);
- /* Parent. */
+ throw SysError(format("executing `%1%'") % program);
+ });
pipe.writeSide.close();
@@ -901,7 +938,7 @@ string runProgram(Path program, bool searchPath, const Strings & args)
/* Wait for the child to finish. */
int status = pid.wait(true);
if (!statusOk(status))
- throw Error(format("program `%1%' %2%")
+ throw ExecError(format("program `%1%' %2%")
% program % statusToString(status));
return result;
@@ -928,13 +965,6 @@ void closeOnExec(int fd)
}
-#if HAVE_VFORK
-pid_t (*maybeVfork)() = vfork;
-#else
-pid_t (*maybeVfork)() = fork;
-#endif
-
-
//////////////////////////////////////////////////////////////////////
diff --git a/nix/libutil/util.hh b/nix/libutil/util.hh
index ce2d77c19a..6a84ed8851 100644
--- a/nix/libutil/util.hh
+++ b/nix/libutil/util.hh
@@ -7,6 +7,7 @@
#include <dirent.h>
#include <unistd.h>
#include <signal.h>
+#include <functional>
#include <cstdio>
@@ -63,7 +64,20 @@ bool isLink(const Path & path);
/* Read the contents of a directory. The entries `.' and `..' are
removed. */
-Strings readDirectory(const Path & path);
+struct DirEntry
+{
+ string name;
+ ino_t ino;
+ unsigned char type; // one of DT_*
+ DirEntry(const string & name, ino_t ino, unsigned char type)
+ : name(name), ino(ino), type(type) { }
+};
+
+typedef vector<DirEntry> DirEntries;
+
+DirEntries readDirectory(const Path & path);
+
+unsigned char getFileType(const Path & path);
/* Read the contents of a file into a string. */
string readFile(int fd);
@@ -157,6 +171,7 @@ extern void (*_writeToStderr) (const unsigned char * buf, size_t count);
requested number of bytes. */
void readFull(int fd, unsigned char * buf, size_t count);
void writeFull(int fd, const unsigned char * buf, size_t count);
+void writeFull(int fd, const string & s);
MakeError(EndOfFile, Error)
@@ -237,10 +252,11 @@ class Pid
int killSignal;
public:
Pid();
+ Pid(pid_t pid);
~Pid();
void operator =(pid_t pid);
operator pid_t();
- void kill();
+ void kill(bool quiet = false);
int wait(bool block);
void setSeparatePG(bool separatePG);
void setKillSignal(int signal);
@@ -252,11 +268,24 @@ public:
void killUser(uid_t uid);
+/* Fork a process that runs the given function, and return the child
+ pid to the caller. */
+pid_t startProcess(std::function<void()> fun, bool dieWithParent = true,
+ const string & errorPrefix = "error: ", bool runExitHandlers = false);
+
+
/* Run a program and return its stdout in a string (i.e., like the
shell backtick operator). */
string runProgram(Path program, bool searchPath = false,
const Strings & args = Strings());
+MakeError(ExecError, Error)
+
+/* Convert a list of strings to a null-terminated vector of char
+ *'s. The result must not be accessed beyond the lifetime of the
+ list of strings. */
+std::vector<const char *> stringsToCharPtrs(const Strings & ss);
+
/* Close all file descriptors except stdin, stdout, stderr, and those
listed in the given set. Good practice in child processes. */
void closeMostFDs(const set<int> & exceptions);
@@ -264,9 +293,6 @@ void closeMostFDs(const set<int> & exceptions);
/* Set the close-on-exec flag for the given file descriptor. */
void closeOnExec(int fd);
-/* Call vfork() if available, otherwise fork(). */
-extern pid_t (*maybeVfork)();
-
/* User interruption. */
diff --git a/nix/nix-daemon/guix-daemon.cc b/nix/nix-daemon/guix-daemon.cc
index f096ed5a97..1934487d24 100644
--- a/nix/nix-daemon/guix-daemon.cc
+++ b/nix/nix-daemon/guix-daemon.cc
@@ -33,6 +33,9 @@
#include <strings.h>
#include <exception>
+#include <libintl.h>
+#include <locale.h>
+
/* Variables used by `nix-daemon.cc'. */
volatile ::sig_atomic_t blockInt;
char **argvSaved;
@@ -45,16 +48,21 @@ extern void run (Strings args);
/* Command-line options. */
+#define n_(str) str
+#define _(str) gettext (str)
+static const char guix_textdomain[] = "guix";
+
+
const char *argp_program_version =
"guix-daemon (" PACKAGE_NAME ") " PACKAGE_VERSION;
const char *argp_program_bug_address = PACKAGE_BUGREPORT;
static char doc[] =
-"guix-daemon -- perform derivation builds and store accesses\
-\v\
-This program is a daemon meant to run in the background. It serves \
+ n_("guix-daemon -- perform derivation builds and store accesses")
+ "\v\n"
+ n_("This program is a daemon meant to run in the background. It serves \
requests sent over a Unix-domain socket. It accesses the store, and \
-builds derivations on behalf of its clients.";
+builds derivations on behalf of its clients.");
#define GUIX_OPT_SYSTEM 1
#define GUIX_OPT_DISABLE_CHROOT 2
@@ -75,56 +83,59 @@ builds derivations on behalf of its clients.";
static const struct argp_option options[] =
{
- { "system", GUIX_OPT_SYSTEM, "SYSTEM", 0,
- "Assume SYSTEM as the current system type" },
- { "cores", 'c', "N", 0,
- "Use N CPU cores to build each derivation; 0 means as many as available" },
- { "max-jobs", 'M', "N", 0,
- "Allow at most N build jobs" },
+ { "system", GUIX_OPT_SYSTEM, n_("SYSTEM"), 0,
+ n_("assume SYSTEM as the current system type") },
+ { "cores", 'c', n_("N"), 0,
+ n_("use N CPU cores to build each derivation; 0 means as many as available")
+ },
+ { "max-jobs", 'M', n_("N"), 0,
+ n_("allow at most N build jobs") },
{ "disable-chroot", GUIX_OPT_DISABLE_CHROOT, 0, 0,
- "Disable chroot builds" },
- { "chroot-directory", GUIX_OPT_CHROOT_DIR, "DIR", 0,
- "Add DIR to the build chroot" },
- { "build-users-group", GUIX_OPT_BUILD_USERS_GROUP, "GROUP", 0,
- "Perform builds as a user of GROUP" },
+ n_("disable chroot builds") },
+ { "chroot-directory", GUIX_OPT_CHROOT_DIR, n_("DIR"), 0,
+ n_("add DIR to the build chroot") },
+ { "build-users-group", GUIX_OPT_BUILD_USERS_GROUP, n_("GROUP"), 0,
+ n_("perform builds as a user of GROUP") },
{ "no-substitutes", GUIX_OPT_NO_SUBSTITUTES, 0, 0,
- "Do not use substitutes" },
- { "substitute-urls", GUIX_OPT_SUBSTITUTE_URLS, "URLS", 0,
- "Use URLS as the default list of substitute providers" },
+ n_("do not use substitutes") },
+ { "substitute-urls", GUIX_OPT_SUBSTITUTE_URLS, n_("URLS"), 0,
+ n_("use URLS as the default list of substitute providers") },
{ "no-build-hook", GUIX_OPT_NO_BUILD_HOOK, 0, 0,
- "Do not use the 'build hook'" },
+ n_("do not use the 'build hook'") },
{ "cache-failures", GUIX_OPT_CACHE_FAILURES, 0, 0,
- "Cache build failures" },
+ n_("cache build failures") },
{ "lose-logs", GUIX_OPT_LOSE_LOGS, 0, 0,
- "Do not keep build logs" },
+ n_("do not keep build logs") },
{ "disable-log-compression", GUIX_OPT_DISABLE_LOG_COMPRESSION, 0, 0,
- "Disable compression of the build logs" },
+ n_("disable compression of the build logs") },
/* '--disable-deduplication' was known as '--disable-store-optimization'
up to Guix 0.7 included, so keep the alias around. */
{ "disable-deduplication", GUIX_OPT_DISABLE_DEDUPLICATION, 0, 0,
- "Disable automatic file \"deduplication\" in the store" },
+ n_("disable automatic file \"deduplication\" in the store") },
{ "disable-store-optimization", GUIX_OPT_DISABLE_DEDUPLICATION, 0,
OPTION_ALIAS | OPTION_HIDDEN, NULL },
- { "impersonate-linux-2.6", GUIX_OPT_IMPERSONATE_LINUX_26, 0, 0,
- "Impersonate Linux 2.6"
-#ifndef HAVE_SYS_PERSONALITY_H
- " (this option has no effect in this configuration)"
+ { "impersonate-linux-2.6", GUIX_OPT_IMPERSONATE_LINUX_26, 0,
+#ifdef HAVE_SYS_PERSONALITY_H
+ 0,
+#else
+ OPTION_HIDDEN,
#endif
+ n_("impersonate Linux 2.6")
},
{ "gc-keep-outputs", GUIX_OPT_GC_KEEP_OUTPUTS,
"yes/no", OPTION_ARG_OPTIONAL,
- "Tell whether the GC must keep outputs of live derivations" },
+ n_("tell whether the GC must keep outputs of live derivations") },
{ "gc-keep-derivations", GUIX_OPT_GC_KEEP_DERIVATIONS,
"yes/no", OPTION_ARG_OPTIONAL,
- "Tell whether the GC must keep derivations corresponding \
-to live outputs" },
+ n_("tell whether the GC must keep derivations corresponding \
+to live outputs") },
- { "listen", GUIX_OPT_LISTEN, "SOCKET", 0,
- "Listen for connections on SOCKET" },
+ { "listen", GUIX_OPT_LISTEN, n_("SOCKET"), 0,
+ n_("listen for connections on SOCKET") },
{ "debug", GUIX_OPT_DEBUG, 0, 0,
- "Produce debugging output" },
+ n_("produce debugging output") },
{ 0, 0, 0, 0, 0 }
};
@@ -154,8 +165,18 @@ parse_opt (int key, char *arg, struct argp_state *state)
settings.useChroot = false;
break;
case GUIX_OPT_CHROOT_DIR:
- settings.dirsInChroot.insert (arg);
- break;
+ {
+ std::string chroot_dirs;
+
+ chroot_dirs = settings.get ("build-extra-chroot-dirs",
+ (std::string) "");
+ if (chroot_dirs == "")
+ chroot_dirs = arg;
+ else
+ chroot_dirs = chroot_dirs + " " + arg;
+ settings.set("build-extra-chroot-dirs", chroot_dirs);
+ break;
+ }
case GUIX_OPT_DISABLE_LOG_COMPRESSION:
settings.compressLog = false;
break;
@@ -181,7 +202,7 @@ parse_opt (int key, char *arg, struct argp_state *state)
}
catch (std::exception &e)
{
- fprintf (stderr, "error: %s\n", e.what ());
+ fprintf (stderr, _("error: %s\n"), e.what ());
exit (EXIT_FAILURE);
}
break;
@@ -220,19 +241,29 @@ parse_opt (int key, char *arg, struct argp_state *state)
}
/* Argument parsing. */
-static struct argp argp = { options, parse_opt, 0, doc };
+static const struct argp argp =
+ {
+ options, parse_opt,
+ NULL, doc,
+ NULL, NULL, // children and help_filter
+ guix_textdomain
+ };
int
main (int argc, char *argv[])
{
- Strings nothing;
+ static const Strings nothing;
+
+ setlocale (LC_ALL, "");
+ bindtextdomain (guix_textdomain, LOCALEDIR);
+ textdomain (guix_textdomain);
/* Initialize libgcrypt. */
if (!gcry_check_version (GCRYPT_VERSION))
{
- fprintf (stderr, "error: libgcrypt version mismatch\n");
+ fprintf (stderr, _("error: libgcrypt version mismatch\n"));
exit (EXIT_FAILURE);
}
@@ -323,16 +354,17 @@ main (int argc, char *argv[])
settings.update ();
if (geteuid () == 0 && settings.buildUsersGroup.empty ())
- fprintf (stderr, "warning: daemon is running as root, so "
- "using `--build-users-group' is highly recommended\n");
+ fprintf (stderr, _("warning: daemon is running as root, so \
+using `--build-users-group' is highly recommended\n"));
if (settings.useChroot)
{
- foreach (PathSet::iterator, i, settings.dirsInChroot)
- {
- printMsg (lvlDebug,
- format ("directory `%1%' added to the chroot") % *i);
- }
+ std::string chroot_dirs;
+
+ chroot_dirs = settings.get ("build-extra-chroot-dirs",
+ (std::string) "");
+ printMsg (lvlDebug,
+ format ("extra chroot directories: '%1%'") % chroot_dirs);
}
printMsg (lvlDebug,
@@ -346,7 +378,7 @@ main (int argc, char *argv[])
}
catch (std::exception &e)
{
- fprintf (stderr, "error: %s\n", e.what ());
+ fprintf (stderr, _("error: %s\n"), e.what ());
return EXIT_FAILURE;
}
diff --git a/nix/nix-daemon/nix-daemon.cc b/nix/nix-daemon/nix-daemon.cc
index 8814fe3155..10159db62e 100644
--- a/nix/nix-daemon/nix-daemon.cc
+++ b/nix/nix-daemon/nix-daemon.cc
@@ -7,6 +7,8 @@
#include "affinity.hh"
#include "globals.hh"
+#include <algorithm>
+
#include <cstring>
#include <unistd.h>
#include <signal.h>
@@ -17,6 +19,8 @@
#include <sys/un.h>
#include <fcntl.h>
#include <errno.h>
+#include <pwd.h>
+#include <grp.h>
using namespace nix;
@@ -44,7 +48,6 @@ static FdSource from(STDIN_FILENO);
static FdSink to(STDOUT_FILENO);
bool canSendStderr;
-pid_t myPid;
@@ -54,11 +57,7 @@ pid_t myPid;
socket. */
static void tunnelStderr(const unsigned char * buf, size_t count)
{
- /* Don't send the message to the client if we're a child of the
- process handling the connection. Otherwise we could screw up
- the protocol. It's up to the parent to redirect stderr and
- send it to the client somehow (e.g., as in build.cc). */
- if (canSendStderr && myPid == getpid()) {
+ if (canSendStderr) {
try {
writeInt(STDERR_NEXT, to);
writeString(buf, count, to);
@@ -284,15 +283,6 @@ static void performOp(bool trusted, unsigned int clientVersion,
{
switch (op) {
-#if 0
- case wopQuit: {
- /* Close the database. */
- store.reset((StoreAPI *) 0);
- writeInt(1, to);
- break;
- }
-#endif
-
case wopIsValidPath: {
/* 'readStorePath' could raise an error leading to the connection
being closed. To be able to recover from an invalid path error,
@@ -450,6 +440,9 @@ static void performOp(bool trusted, unsigned int clientVersion,
case wopImportPaths: {
startWork();
TunnelSource source(from);
+
+ /* Unlike Nix, always require a signature, even for "trusted"
+ users. */
Paths paths = store->importPaths(true, source);
stopWork();
writeStrings(paths, to);
@@ -650,6 +643,25 @@ static void performOp(bool trusted, unsigned int clientVersion,
break;
}
+ case wopOptimiseStore:
+ startWork();
+ store->optimiseStore();
+ stopWork();
+ writeInt(1, to);
+ break;
+
+ case wopVerifyStore: {
+ bool checkContents = readInt(from) != 0;
+ bool repair = readInt(from) != 0;
+ startWork();
+ if (repair && !trusted)
+ throw Error("you are not privileged to repair paths");
+ bool errors = store->verifyStore(checkContents, repair);
+ stopWork();
+ writeInt(errors, to);
+ break;
+ }
+
default:
throw Error(format("invalid operation %1%") % op);
}
@@ -659,7 +671,6 @@ static void performOp(bool trusted, unsigned int clientVersion,
static void processConnection(bool trusted)
{
canSendStderr = false;
- myPid = getpid();
_writeToStderr = tunnelStderr;
#ifdef HAVE_HUP_NOTIFICATION
@@ -708,7 +719,7 @@ static void processConnection(bool trusted)
to.flush();
} catch (Error & e) {
- stopWork(false, e.msg());
+ stopWork(false, e.msg(), GET_PROTOCOL_MINOR(clientVersion) >= 8 ? 1 : 0);
to.flush();
return;
}
@@ -735,12 +746,10 @@ static void processConnection(bool trusted)
during addTextToStore() / importPath(). If that
happens, just send the error message and exit. */
bool errorAllowed = canSendStderr;
- if (!errorAllowed) printMsg(lvlError, format("error processing client input: %1%") % e.msg());
stopWork(false, e.msg(), GET_PROTOCOL_MINOR(clientVersion) >= 8 ? e.status : 0);
- if (!errorAllowed) break;
+ if (!errorAllowed) throw;
} catch (std::bad_alloc & e) {
- if (canSendStderr)
- stopWork(false, "Nix daemon out of memory", GET_PROTOCOL_MINOR(clientVersion) >= 8 ? 1 : 0);
+ stopWork(false, "Nix daemon out of memory", GET_PROTOCOL_MINOR(clientVersion) >= 8 ? 1 : 0);
throw;
}
@@ -749,7 +758,9 @@ static void processConnection(bool trusted)
assert(!canSendStderr);
};
- printMsg(lvlError, format("%1% operations") % opCount);
+ canSendStderr = false;
+ _isInterrupted = false;
+ printMsg(lvlDebug, format("%1% operations") % opCount);
}
@@ -771,11 +782,35 @@ static void setSigChldAction(bool autoReap)
}
+bool matchUser(const string & user, const string & group, const Strings & users)
+{
+ if (find(users.begin(), users.end(), "*") != users.end())
+ return true;
+
+ if (find(users.begin(), users.end(), user) != users.end())
+ return true;
+
+ for (auto & i : users)
+ if (string(i, 0, 1) == "@") {
+ if (group == string(i, 1)) return true;
+ struct group * gr = getgrnam(i.c_str() + 1);
+ if (!gr) continue;
+ for (char * * mem = gr->gr_mem; *mem; mem++)
+ if (user == string(*mem)) return true;
+ }
+
+ return false;
+}
+
+
#define SD_LISTEN_FDS_START 3
static void daemonLoop()
{
+ if (chdir("/") == -1)
+ throw SysError("cannot change current directory");
+
/* Get rid of children automatically; don't let them become
zombies. */
setSigChldAction(true);
@@ -804,7 +839,8 @@ static void daemonLoop()
/* Urgh, sockaddr_un allows path names of only 108 characters.
So chdir to the socket directory so that we can pass a
relative path name. */
- chdir(dirOf(socketPath).c_str());
+ if (chdir(dirOf(socketPath).c_str()) == -1)
+ throw SysError("cannot change current directory");
Path socketPathRel = "./" + baseNameOf(socketPath);
struct sockaddr_un addr;
@@ -824,7 +860,8 @@ static void daemonLoop()
if (res == -1)
throw SysError(format("cannot bind to socket `%1%'") % socketPath);
- chdir("/"); /* back to the root */
+ if (chdir("/") == -1) /* back to the root */
+ throw SysError("cannot change current directory");
if (listen(fdSocket, 5) == -1)
throw SysError(format("cannot listen on socket `%1%'") % socketPath);
@@ -856,58 +893,61 @@ static void daemonLoop()
closeOnExec(remote);
- /* Get the identity of the caller, if possible. */
- uid_t clientUid = -1;
- pid_t clientPid = -1;
bool trusted = false;
+ pid_t clientPid = -1;
#if defined(SO_PEERCRED)
+ /* Get the identity of the caller, if possible. */
ucred cred;
socklen_t credLen = sizeof(cred);
- if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) != -1) {
- clientPid = cred.pid;
- clientUid = cred.uid;
- if (clientUid == 0) trusted = true;
- }
-#endif
+ if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == -1)
+ throw SysError("getting peer credentials");
- printMsg(lvlInfo, format("accepted connection from pid %1%, uid %2%") % clientPid % clientUid);
+ clientPid = cred.pid;
- /* Fork a child to handle the connection. */
- pid_t child;
- child = fork();
+ struct passwd * pw = getpwuid(cred.uid);
+ string user = pw ? pw->pw_name : int2String(cred.uid);
- switch (child) {
+ struct group * gr = getgrgid(cred.gid);
+ string group = gr ? gr->gr_name : int2String(cred.gid);
- case -1:
- throw SysError("unable to fork");
+ Strings trustedUsers = settings.get("trusted-users", Strings({"root"}));
+ Strings allowedUsers = settings.get("allowed-users", Strings({"*"}));
- case 0:
- try { /* child */
+ if (matchUser(user, group, trustedUsers))
+ trusted = true;
- /* Background the daemon. */
- if (setsid() == -1)
- throw SysError(format("creating a new session"));
+ if (!trusted && !matchUser(user, group, allowedUsers))
+ throw Error(format("user `%1%' is not allowed to connect to the Nix daemon") % user);
- /* Restore normal handling of SIGCHLD. */
- setSigChldAction(false);
+ printMsg(lvlInfo, format((string) "accepted connection from pid %1%, user %2%"
+ + (trusted ? " (trusted)" : "")) % clientPid % user);
+#endif
- /* For debugging, stuff the pid into argv[1]. */
- if (clientPid != -1 && argvSaved[1]) {
- string processName = int2String(clientPid);
- strncpy(argvSaved[1], processName.c_str(), strlen(argvSaved[1]));
- }
+ /* Fork a child to handle the connection. */
+ startProcess([&]() {
+ fdSocket.close();
- /* Handle the connection. */
- from.fd = remote;
- to.fd = remote;
- processConnection(trusted);
+ /* Background the daemon. */
+ if (setsid() == -1)
+ throw SysError(format("creating a new session"));
- } catch (std::exception & e) {
- writeToStderr("unexpected Nix daemon error: " + string(e.what()) + "\n");
+ /* Restore normal handling of SIGCHLD. */
+ setSigChldAction(false);
+
+ /* For debugging, stuff the pid into argv[1]. */
+ if (clientPid != -1 && argvSaved[1]) {
+ string processName = int2String(clientPid);
+ strncpy(argvSaved[1], processName.c_str(), strlen(argvSaved[1]));
}
+
+ /* Handle the connection. */
+ from.fd = remote;
+ to.fd = remote;
+ processConnection(trusted);
+
exit(0);
- }
+ }, false, "unexpected Nix daemon error: ", true);
} catch (Interrupted & e) {
throw;
@@ -925,7 +965,6 @@ void run(Strings args)
if (arg == "--daemon") /* ignored for backwards compatibility */;
}
- chdir("/");
daemonLoop();
}
generated by cgit v1.2.3 (git 2.46.2) at 2024-11-11 11:51:46 +0000