Permalink
Browse files

Change the way spawn errors are handled.

When a spawn error occurs, log full process environment to a log file.
Spawn errors are now logged right after they are created, not from the RequestHandler.

Closes GH-1175.
Closes GH-1021.
  • Loading branch information...
FooBarWidget committed Jun 3, 2014
1 parent 76b9250 commit deab76c5c9ef217df4c32f291ff8142800d114ed
View
@@ -8,6 +8,9 @@ Release 4.0.46
modules from the same directory.
* Fixed a compatibility problem with Django, which could cause Django apps to
freeze indefinitely. Closes GH-1215.
* Logging of application spawning errors has been much improved. Full details
about the error, such as environment variables, are saved to a private log file.
In the past, these details were only viewable in the browser.
* Passenger Standalone no longer, by default, loads shell startup files before
loading the application. This is because Passenger Standalone is often invoked
from the shell anyway. Indeed, loading shell startup files again can interfere
View
@@ -84,6 +84,7 @@
'ext/common/ApplicationPool2/SpawnerFactory.h',
'ext/common/ApplicationPool2/SmartSpawner.h',
'ext/common/ApplicationPool2/DirectSpawner.h',
'ext/common/ApplicationPool2/ErrorRenderer.h',
LIBBOOST_OXT,
helper_agent_libs.link_objects,
LIBEV_TARGET,
@@ -29,6 +29,7 @@
#include <boost/shared_ptr.hpp>
#include <boost/function.hpp>
#include <oxt/tracable_exception.hpp>
#include <ResourceLocator.h>
#include <RandomGenerator.h>
#include <ApplicationPool2/Options.h>
#include <Utils/StringMap.h>
@@ -183,6 +184,9 @@ struct Ticket {
};
struct SpawnerConfig {
// Used by error pages.
ResourceLocator resourceLocator;
// Used by SmartSpawner and DirectSpawner.
/** A random generator to use. */
RandomGeneratorPtr randomGenerator;
@@ -192,8 +196,10 @@ struct SpawnerConfig {
unsigned int spawnerCreationSleepTime;
unsigned int spawnTime;
SpawnerConfig(const RandomGeneratorPtr &randomGenerator = RandomGeneratorPtr())
: concurrency(1),
SpawnerConfig(const ResourceLocator &_resourceLocator,
const RandomGeneratorPtr &randomGenerator = RandomGeneratorPtr())
: resourceLocator(_resourceLocator),
concurrency(1),
spawnerCreationSleepTime(0),
spawnTime(0)
{
@@ -209,6 +215,8 @@ typedef boost::shared_ptr<SpawnerConfig> SpawnerConfigPtr;
ExceptionPtr copyException(const tracable_exception &e);
void rethrowException(const ExceptionPtr &e);
void processAndLogNewSpawnException(SpawnException &e, const Options &options,
ResourceLocator &resourceLocator, RandomGenerator &randomGenerator);
} // namespace ApplicationPool2
} // namespace Passenger
@@ -111,10 +111,10 @@ class DirectSpawner: public Spawner {
shared_array<const char *> &args) const
{
vector<string> startCommandArgs;
string agentsDir = resourceLocator.getAgentsDir();
string agentsDir = config->resourceLocator.getAgentsDir();
vector<string> command;
split(options.getStartCommand(resourceLocator), '\t', startCommandArgs);
split(options.getStartCommand(config->resourceLocator), '\t', startCommandArgs);
if (startCommandArgs.empty()) {
throw RuntimeException("No startCommand given");
}
@@ -144,17 +144,11 @@ class DirectSpawner: public Spawner {
}
public:
DirectSpawner(const ResourceLocator &_resourceLocator,
const ServerInstanceDir::GenerationPtr &_generation,
const SpawnerConfigPtr &_config = SpawnerConfigPtr())
: Spawner(_resourceLocator)
DirectSpawner(const ServerInstanceDir::GenerationPtr &_generation,
const SpawnerConfigPtr &_config)
: Spawner(_config)
{
generation = _generation;
if (_config == NULL) {
config = boost::make_shared<SpawnerConfig>();
} else {
config = _config;
}
}
virtual ProcessPtr spawn(const Options &options) {
@@ -37,16 +37,14 @@ using namespace oxt;
class DummySpawner: public Spawner {
private:
SpawnerConfigPtr config;
boost::mutex lock;
unsigned int count;
public:
unsigned int cleanCount;
DummySpawner(const ResourceLocator &resourceLocator, const SpawnerConfigPtr &_config)
: Spawner(resourceLocator),
config(_config)
DummySpawner(const SpawnerConfigPtr &_config)
: Spawner(_config)
{
count = 0;
cleanCount = 0;
@@ -0,0 +1,119 @@
/*
* Phusion Passenger - https://www.phusionpassenger.com/
* Copyright (c) 2014 Phusion
*
* "Phusion Passenger" is a trademark of Hongli Lai & Ninh Bui.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#ifndef _PASSENGER_APPLICATION_POOL2_ERROR_RENDERER_H_
#define _PASSENGER_APPLICATION_POOL2_ERROR_RENDERER_H_
#include <string>
#include <map>
#include <cctype>
#include <ApplicationPool2/Options.h>
#include <Constants.h>
#include <ResourceLocator.h>
#include <StaticString.h>
#include <Exceptions.h>
#include <Utils/StringMap.h>
#include <Utils/Template.h>
#include <Utils/IOUtils.h>
namespace Passenger {
namespace ApplicationPool2 {
using namespace std;
using namespace boost;
using namespace oxt;
class ErrorRenderer {
private:
string templatesDir, cssFile, errorLayoutFile;
public:
ErrorRenderer(const ResourceLocator &resourceLocator) {
templatesDir = resourceLocator.getResourcesDir() + "/templates";
cssFile = templatesDir + "/error_layout.css";
errorLayoutFile = templatesDir + "/error_layout.html.template";
}
string renderWithDetails(const StaticString &message,
const Options &options,
const SpawnException *e = NULL) const
{
string generalErrorFile =
(e != NULL && e->isHTML())
? templatesDir + "/general_error_with_html.html.template"
: templatesDir + "/general_error.html.template";
string css = readAll(cssFile);
StringMap<StaticString> params;
params.set("CSS", css);
params.set("APP_ROOT", options.appRoot);
params.set("RUBY", options.ruby);
params.set("ENVIRONMENT", options.environment);
params.set("MESSAGE", message);
params.set("IS_RUBY_APP",
(options.appType == "classic-rails" || options.appType == "rack")
? "true" : "false");
if (e != NULL) {
params.set("TITLE", "Web application could not be started");
// Store all SpawnException annotations into 'params',
// but convert its name to uppercase.
const map<string, string> &annotations = e->getAnnotations();
map<string, string>::const_iterator it, end = annotations.end();
for (it = annotations.begin(); it != end; it++) {
string name = it->first;
for (string::size_type i = 0; i < name.size(); i++) {
name[i] = toupper(name[i]);
}
params.set(name, it->second);
}
} else {
params.set("TITLE", "Internal server error");
}
string content = Template::apply(readAll(generalErrorFile), params);
params.set("CONTENT", content);
return Template::apply(readAll(errorLayoutFile), params);
}
string renderWithoutDetails() const {
string templateFile = templatesDir + "/undisclosed_error.html.template";
StringMap<StaticString> params;
params.set("PROGRAM_NAME", PROGRAM_NAME);
params.set("NGINX_DOC_URL", NGINX_DOC_URL);
params.set("APACHE2_DOC_URL", APACHE2_DOC_URL);
params.set("STANDALONE_DOC_URL", STANDALONE_DOC_URL);
return Template::apply(readAll(templateFile), params);
}
};
} // namespace ApplicationPool2
} // namespace Passenger
#endif /* _PASSENGER_APPLICATION_POOL2_ERROR_RENDERER_H_ */
@@ -25,15 +25,19 @@
#include <typeinfo>
#include <algorithm>
#include <utility>
#include <sstream>
#include <limits.h>
#include <boost/make_shared.hpp>
#include <boost/date_time/posix_time/posix_time_types.hpp>
#include <oxt/backtrace.hpp>
#include <ApplicationPool2/Pool.h>
#include <ApplicationPool2/SuperGroup.h>
#include <ApplicationPool2/Group.h>
#include <ApplicationPool2/PipeWatcher.h>
#include <ApplicationPool2/ErrorRenderer.h>
#include <Exceptions.h>
#include <MessageReadersWriters.h>
#include <Utils.h>
#include <Utils/ScopeGuard.h>
#include <Utils/MessageIO.h>
@@ -136,6 +140,59 @@ rethrowException(const ExceptionPtr &e) {
throw tracable_exception(*e);
}
void processAndLogNewSpawnException(SpawnException &e, const Options &options,
ResourceLocator &resourceLocator, RandomGenerator &randomGenerator)
{
ErrorRenderer renderer(resourceLocator);
string errorId = randomGenerator.generateHexString(4);
string appMessage = e.getErrorPage();
char filename[PATH_MAX];
stringstream stream;
e.set("ERROR_ID", errorId);
if (appMessage.empty()) {
appMessage = "none";
}
try {
int fd = -1;
FdGuard guard(fd, true);
string errorPage;
errorPage = renderer.renderWithDetails(appMessage, options, &e);
#if defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__)
snprintf(filename, PATH_MAX, "%s/passenger-error-XXXXXX.html",
getSystemTempDir());
fd = mkstemps(filename, sizeof(".html") - 1);
#else
snprintf(filename, PATH_MAX, "%s/passenger-error.XXXXXX",
getSystemTempDir());
fd = mkstemp(filename);
#endif
if (fd == -1) {
int e = errno;
throw SystemException("Cannot generate a temporary filename",
e);
}
writeExact(fd, errorPage);

This comment has been minimized.

Show comment
Hide comment
@paulRbr

paulRbr Dec 14, 2014

Contributor

Hello @FooBarWidget !

Shouldn't we close the file descriptor at this point? (I've noticed this after some ULIMIT error when a service couldn't start) Let me know, if I'm correct, to open a PR for that if needed be.

@paulRbr

paulRbr Dec 14, 2014

Contributor

Hello @FooBarWidget !

Shouldn't we close the file descriptor at this point? (I've noticed this after some ULIMIT error when a service couldn't start) Let me know, if I'm correct, to open a PR for that if needed be.

This comment has been minimized.

Show comment
Hide comment
@FooBarWidget

FooBarWidget Dec 14, 2014

Member

It looks like there's a bug there. I should create the FdGuard after assigning the fd. :(

@FooBarWidget

FooBarWidget Dec 14, 2014

Member

It looks like there's a bug there. I should create the FdGuard after assigning the fd. :(

This comment has been minimized.

Show comment
Hide comment
@FooBarWidget

FooBarWidget Dec 14, 2014

Member

I've filed an issue here: #1324

@FooBarWidget

FooBarWidget Dec 14, 2014

Member

I've filed an issue here: #1324

} catch (const SystemException &e2) {
filename[0] = '\0';
P_ERROR("Cannot render an error page: " << e2.what() << "\n" <<
e2.backtrace());
}
stream << "Could not spawn process for application " << options.appRoot <<
": " << e.what() << "\n" <<
" Error ID: " << errorId << "\n";
if (filename[0] != '\0') {
stream << " Error details saved to: " << filename << "\n";
}
stream << " Message from application: " << appMessage << "\n";
P_ERROR(stream.str());
}
const SuperGroupPtr
Pool::getSuperGroup(const char *name) {
@@ -187,6 +244,8 @@ SuperGroup::realDoInitialize(const Options &options, unsigned int generation) {
vector<ComponentInfo> componentInfos;
vector<ComponentInfo>::const_iterator it;
ExceptionPtr exception;
PoolPtr pool = getPool();
P_TRACE(2, "Initializing SuperGroup " << inspect() << " in the background...");
try {
@@ -198,13 +257,15 @@ SuperGroup::realDoInitialize(const Options &options, unsigned int generation) {
string message = "The directory " +
options.appRoot +
" does not seem to contain a web application.";
exception = boost::make_shared<SpawnException>(
message, message, false);
boost::shared_ptr<SpawnException> spawnException =
boost::make_shared<SpawnException>(
message, message, false);
exception = spawnException;
processAndLogNewSpawnException(*spawnException, options,
pool->getResourceLocator(), *pool->randomGenerator);
}
PoolPtr pool = getPool();
Pool::DebugSupportPtr debug = pool->debugSupport;
vector<Callback> actions;
{
if (debug != NULL && debug->superGroup) {
@@ -805,7 +866,10 @@ Group::spawnThreadRealMain(const SpawnerPtr &spawner, const Options &options, un
this_thread::restore_interruption ri(di);
this_thread::restore_syscall_interruption rsi(dsi);
if (shouldFail) {
throw SpawnException("Simulated failure");
SpawnException e("Simulated failure");
processAndLogNewSpawnException(e, options, pool->getResourceLocator(),
*pool->randomGenerator);
throw e;
} else {
process = spawner->spawn(options);
process->setGroup(shared_from_this());
@@ -878,9 +942,6 @@ Group::spawnThreadRealMain(const SpawnerPtr &spawner, const Options &options, un
} else {
// TODO: sure this is the best thing? if there are
// processes currently alive we should just use them.
P_ERROR("Could not spawn process for group " << name <<
": " << exception->what() << "\n" <<
exception->backtrace());
if (enabledCount == 0) {
enableAllDisablingProcesses(actions);
}
@@ -1197,7 +1258,7 @@ Group::testOverflowRequestQueue() const {
const ResourceLocator &
Group::getResourceLocator() const {
return getPool()->spawnerFactory->getResourceLocator();
return getPool()->getResourceLocator();
}
// 'process' is not a reference so that bind(runAttachHooks, ...) causes the shared
@@ -279,6 +279,10 @@ class Pool: public boost::enable_shared_from_this<Pool> {
}
}
ResourceLocator &getResourceLocator() const {
return spawnerFactory->getConfig()->resourceLocator;
}
ProcessPtr findOldestIdleProcess(const Group *exclude = NULL) const {
ProcessPtr oldestIdleProcess;
Oops, something went wrong.

0 comments on commit deab76c

Please sign in to comment.