Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Site updated at 2012-08-31 05:30:06 UTC

  • Loading branch information...
commit 3cb7bc6f6bbf495a8773cacd1d0c038f4f28d1e9 1 parent 44bf5e5
@dcramer authored
View
2  2012/08/30/how-noops-works-for-sentry/index.html
@@ -231,7 +231,7 @@ <h1 class="title">Moving Sentry From Heroku to Hardware</h1>
<div class="cat">
- <a class='category' href='/categories/django/'>django</a>, <a class='category' href='/categories/ops/'>ops</a>, <a class='category' href='/categories/sentry/'>sentry</a>
+ <a class='category' href='/categories/django/'>django</a>, <a class='category' href='/categories/heroku/'>heroku</a>, <a class='category' href='/categories/ops/'>ops</a>, <a class='category' href='/categories/sentry/'>sentry</a>
</div>
View
4 archives/index.html
@@ -7,7 +7,7 @@
<meta name="author" content="David Cramer">
- <meta name="description" content="Blog Archive 2012 Moving Sentry from Heroku to Hardware Aug 30 django, ops, sentry Comments Scaling Your Clouds Jun 3 django, heroku, sentry &hellip;">
+ <meta name="description" content="Blog Archive 2012 Moving Sentry from Heroku to Hardware Aug 30 django, heroku, ops, sentry Comments Scaling Your Clouds Jun 3 django, heroku, sentry &hellip;">
<meta name="viewport" content="width=device-width; initial-scale=1; maximum-scale=1">
@@ -83,7 +83,7 @@ <h1 class="title"><a href="/2012/08/30/how-noops-works-for-sentry">Moving Sentry
<div class="cat">
- <a class='category' href='/categories/django/'>django</a>, <a class='category' href='/categories/ops/'>ops</a>, <a class='category' href='/categories/sentry/'>sentry</a>
+ <a class='category' href='/categories/django/'>django</a>, <a class='category' href='/categories/heroku/'>heroku</a>, <a class='category' href='/categories/ops/'>ops</a>, <a class='category' href='/categories/sentry/'>sentry</a>
</div>
View
2  atom.xml
@@ -4,7 +4,7 @@
<title><![CDATA[David Cramer's Blog]]></title>
<link href="http://justcramer.com/atom.xml" rel="self"/>
<link href="http://justcramer.com/"/>
- <updated>2012-08-30T22:21:30-07:00</updated>
+ <updated>2012-08-30T22:29:48-07:00</updated>
<id>http://justcramer.com/</id>
<author>
<name><![CDATA[David Cramer]]></name>
View
4 blog/archives/index.html
@@ -7,7 +7,7 @@
<meta name="author" content="David Cramer">
- <meta name="description" content="2012 Moving Sentry from Heroku to Hardware Aug 30 django, ops, sentry Comments Scaling Your Clouds Jun 3 django, heroku, sentry Comments The Cloud &hellip;">
+ <meta name="description" content="2012 Moving Sentry from Heroku to Hardware Aug 30 django, heroku, ops, sentry Comments Scaling Your Clouds Jun 3 django, heroku, sentry Comments The &hellip;">
<meta name="viewport" content="width=device-width; initial-scale=1; maximum-scale=1">
@@ -79,7 +79,7 @@ <h1 class="title"><a href="/2012/08/30/how-noops-works-for-sentry">Moving Sentry
<div class="cat">
- <a class='category' href='/categories/django/'>django</a>, <a class='category' href='/categories/ops/'>ops</a>, <a class='category' href='/categories/sentry/'>sentry</a>
+ <a class='category' href='/categories/django/'>django</a>, <a class='category' href='/categories/heroku/'>heroku</a>, <a class='category' href='/categories/ops/'>ops</a>, <a class='category' href='/categories/sentry/'>sentry</a>
</div>
View
2  categories/ci/atom.xml
@@ -4,7 +4,7 @@
<title><![CDATA[Category: ci | David Cramer's Blog]]></title>
<link href="http://justcramer.com/categories/ci/atom.xml" rel="self"/>
<link href="http://justcramer.com/"/>
- <updated>2012-08-30T22:21:30-07:00</updated>
+ <updated>2012-08-30T22:29:48-07:00</updated>
<id>http://justcramer.com/</id>
<author>
<name><![CDATA[David Cramer]]></name>
View
2  categories/disqus/atom.xml
@@ -4,7 +4,7 @@
<title><![CDATA[Category: disqus | David Cramer's Blog]]></title>
<link href="http://justcramer.com/categories/disqus/atom.xml" rel="self"/>
<link href="http://justcramer.com/"/>
- <updated>2012-08-30T22:21:30-07:00</updated>
+ <updated>2012-08-30T22:29:48-07:00</updated>
<id>http://justcramer.com/</id>
<author>
<name><![CDATA[David Cramer]]></name>
View
2  categories/django/atom.xml
@@ -4,7 +4,7 @@
<title><![CDATA[Category: django | David Cramer's Blog]]></title>
<link href="http://justcramer.com/categories/django/atom.xml" rel="self"/>
<link href="http://justcramer.com/"/>
- <updated>2012-08-30T22:21:30-07:00</updated>
+ <updated>2012-08-30T22:29:48-07:00</updated>
<id>http://justcramer.com/</id>
<author>
<name><![CDATA[David Cramer]]></name>
View
2  categories/django/index.html
@@ -79,7 +79,7 @@ <h1 class="title"><a href="/2012/08/30/how-noops-works-for-sentry">Moving Sentry
<div class="cat">
- <a class='category' href='/categories/django/'>django</a>, <a class='category' href='/categories/ops/'>ops</a>, <a class='category' href='/categories/sentry/'>sentry</a>
+ <a class='category' href='/categories/django/'>django</a>, <a class='category' href='/categories/heroku/'>heroku</a>, <a class='category' href='/categories/ops/'>ops</a>, <a class='category' href='/categories/sentry/'>sentry</a>
</div>
View
2  categories/git/atom.xml
@@ -4,7 +4,7 @@
<title><![CDATA[Category: git | David Cramer's Blog]]></title>
<link href="http://justcramer.com/categories/git/atom.xml" rel="self"/>
<link href="http://justcramer.com/"/>
- <updated>2012-08-30T22:21:30-07:00</updated>
+ <updated>2012-08-30T22:29:48-07:00</updated>
<id>http://justcramer.com/</id>
<author>
<name><![CDATA[David Cramer]]></name>
View
173 categories/heroku/atom.xml
@@ -4,7 +4,7 @@
<title><![CDATA[Category: heroku | David Cramer's Blog]]></title>
<link href="http://justcramer.com/categories/heroku/atom.xml" rel="self"/>
<link href="http://justcramer.com/"/>
- <updated>2012-08-30T22:21:30-07:00</updated>
+ <updated>2012-08-30T22:29:48-07:00</updated>
<id>http://justcramer.com/</id>
<author>
<name><![CDATA[David Cramer]]></name>
@@ -14,6 +14,177 @@
<entry>
+ <title type="html"><![CDATA[Moving Sentry from Heroku to Hardware]]></title>
+ <link href="http://justcramer.com/2012/08/30/how-noops-works-for-sentry"/>
+ <updated>2012-08-30T20:59:00-07:00</updated>
+ <id>http://justcramer.com/2012/08/30/how-noops-works-for-sentry</id>
+ <content type="html"><![CDATA[<p>I've talked a lot about how I run <a href="http://getsentry.com">getsentry.com</a>, mostly with <a href="http://justcramer.com/2012/06/02/the-cloud-is-not-for-you/">my experiences
+on Heroku</a> and how I <a href="http://justcramer.com/2012/06/03/scaling-your-clouds/">switched to leased servers</a>. Many people consistently
+suggested that operations work is difficult so they shouldn't deal with it themselves. I'm not going to tell you that my
+roommate, <a href="http://twitter.com/drunkdev">Mike Clarke</a>, one of the few operations people we have at
+<a href="http://disqus.com">DISQUS</a>, has it easy, but I'd like to give you a little bit of food for thought.</p>
+
+
+
+
+<p>GetSentry started around Christmas of 2011. I had already built and open sourced <a href="http://github.com/getsentry/sentry">Sentry</a> at Disqus, and the idea was to
+ take that work and create a Heroku AddOn out of it. The pitch was that I could make a little bit of money on the side
+ simply by hosting Sentry for people. About three months later I had that prototype hosting service running on Heroku,
+ accepting payments both via the AddOn infrastructure, as well as on my own using the amazing
+ <a href="http://stripe.com">Stripe</a> platform.</p>
+
+
+
+
+<p>Let's fast forward to today. I no longer run any servers on Heroku (or any cloud provider, other than S3 for backups),
+ and instead I lease servers. Now the company I lease from is what most people would call a "budget provider". They're extremely cheap (they dont'
+ add extreme margins to the cost of the machines you're leasing), and they do absolutely nothing for you. It's not
+ for the faint of heart. That said, it's also how I can get away with very low costs.</p>
+
+
+
+
+<p>I'm going to tell you a bit of a story of how I switched from Heroku to fully configured leased servers in less than
+ a week, in my free time. I'm also going to try to convince you that it's really <strong>not that complicated</strong>,.</p>
+
+
+
+
+<h3>The First Server</h3>
+
+
+
+
+<p>This part could be more appropriately titled "Learning Chef". I'm fortunate to have some awesome coworkers, and even
+ more fortunate that when I was making this transitiong I had access to my roommate to prod him about questions. I'm
+ also extremely fortunate that medians like Google, IRC, and Twitter exist for any other questions I ever have.
+
+<p>The first task I had to getting my prototype web server online was to get it all configured. I could have taken the
+ old fashioned approach of creating a few config files locally (vcs maybe) and then sending them up to the server,
+ as well as manually installing whatever packages I needed (nginx, memcache, etc.), but with Puppet and Chef becoming
+ all the range I figured it was as good as time as ever to dig into one.</p>
+
+<p>I decided to use the Chef hosted service, and after a few bumps with figuring out what all this Ruby stuff was about,
+ I had managed to get a basic understanding of roles and cookbooks. After quite a bit of fiddling I had created a
+ cookbook specific to getsentry (which holds things like setting up varoius paths), and a bunch of generic ones,
+ like apt, nginx, memcached, python, etc.</p>
+
+<h3>Creating a Recipe</h3>
+
+<p>The meat of this was handled via Chef's awesome roles, and wiring up a few things in the 'default' recipe of getsentry:</p>
+
+<pre>include_recipe "python"
+
+directory "/srv/www" do
+ owner "root"
+ group "root"
+ mode "0755"
+ action :create
+end
+
+directory "/srv/www/getsentry.com" do
+ owner "dcramer"
+ group "dcramer"
+ mode "0755"
+ action :create
+end
+</pre>
+
+<p>This formed the basis of any server that I would be running, and simply setup a couple of directories. I also simply
+ gave ownership to my user, as I'm the only one working on the project, and didn't need the added complexities of build
+ or system users.</p>
+
+<p>I then moved on to a second recipe, which formed the basis of a web node. This one has a lot more to it, as it needed
+ to configure nginx and memcache at the start:</p>
+
+<pre>include_recipe "getsentry"
+include_recipe "supervisor"
+
+template "#{node[:nginx][:dir]}/sites-available/getsentry.com" do
+ source "nginx/getsentry.erb"
+ owner "root"
+ group "root"
+ mode 0644
+ notifies :reload, "service[nginx]"
+end
+
+nginx_site "getsentry.com"
+
+supervisor_service "web-1" do
+ directory "/srv/www/getsentry.com/current/"
+ command "/srv/www/getsentry.com/env/bin/python manage.py run_gunicorn -b 0.0.0.0:9000 -w #{node[:getsentry][:web][:workers]}"
+ environment "DJANGO_CONF" => node[:django_conf]
+ user "dcramer"
+end
+
+supervisor_service "web-2" do
+ directory "/srv/www/getsentry.com/current/"
+ command "/srv/www/getsentry.com/env/bin/python manage.py run_gunicorn -b 0.0.0.0:9001 -w #{node[:getsentry][:web][:workers]}"
+ environment "DJANGO_CONF" => node[:django_conf]
+ user "dcramer"
+end</pre>
+
+<p>There is a bit more to it then what I've shown, but all in all it was pretty simple. It just took me a bit to understand
+ how chef functioned. All in all, I'm now an engineer that has experience in Chef, even if it's very little. From
+ from my perspective (on the hiring end at Disqus), that's is an awesome addition to an engineer's skillset.</p>
+
+<p>Once the web server was online, all I had to do was to configure a primary database server. I simply brought up another
+ node, gave it a new role (db), and didn't even need to create a custom recipe (I simply reused the existing pgbouncer,
+ postgersql, and redis recipes available elsewhere on the internet).</p>
+
+<h3>Operational Complexity</h3>
+
+<p>I stated in the beginning that I completed this process in less than a week. From Heroku to hardware it took me about
+ three evenings of toying with Chef (mostly more complex components, like iptables and building a deploy script). What
+ I really want to point out is how I have <strong>never</strong> been in an operations position. I've definitely configured
+ servers (ala apt-get install nano), and know my way around, especially with a database, but most of this was fairly
+ new to me.</p>
+
+<p>The continued argument of it being "too difficult" to run your own servers is quite the overstatement, but it's not
+ something you should ignore. There are many things I have to be concerned about, most importantly data loss and the
+ ability to recover in the event of a disaster on my machines. These also aren't overly complex challenges to handle.</p>
+
+<p>Data redundancy is handled a simple cron script that does nightly backups to S3. It's literally just a script that calls
+ pg_dump and s3cmd to send the files upstream. Now that's not enough for any real requirements, so step two is simply
+ setting up replication on your database node to a second server, if if that server is your application server.</p>
+
+<p>Availability is the second big problem, and is easily avoided the same way that you avoid losing your database: have
+ a second server. This again can be a server thats primary task is for something other than your application (it can
+ be your database). It doesnt have to a permanent location for it. It only has to survive until a primary server is
+ available or you're willing and able to invest in more hardware.</p>
+
+<h3>Closing Thoughts</h3>
+
+<p>I spent an initial three evenings, and another week's worth since on server configuring an operations. There were
+ various problems like Postgres not being tuned well enough (pgtune is amazing by the way), DNS being slow (fuck it,
+ use IPs), and some more minor things that needed addressed throughout that time. All in all, there's basically
+ zero day-to-day operations concerns, and most of the work happens when I need to expand the system (which is rare).</p>
+
+<p>All of it ended as an extremely valuable learning experience, but you using Chef wasn't a necessity. I could have done
+ things the more "amateur" way, but I also now have the benefit of being able to bring online a server, run a few
+ commands, and have a machine or even a cluster identical to what's already running.</p>
+
+<p>On the limited hardware I run for getsentry.com, that is, two servers that actually service requests (one database,
+ one app), we've serviced around 25 million requests since August 1st, doing anywhere from 500k to 2 million in a
+ single day. That isn't that much traffic, but what's important is it services those requests very quickly, and is
+ using very little of the resources that are dedicated to it. In the end, this means that Sentry's revenue will grow
+ much more quickly than it's monthly bill will.</p>
+
+<p>GetSentry has been profitable since its 4th month, and currently only spends 10% of its monthly revenue (hardware and
+ other third party services). That gap gets larger every month, and I've been more than happy to invest some of my
+ time to keep that gap as large as possible. The irony of it all? I'm selling a service that's entirely open source,
+ yet suggesting that you run your own hardware. For some people sacrificing cost for convenience is acceptable, for other
+ it may not be.</p>
+
+<p>Also, <a href="http://whoownsmyavailability.com/">this</a>.</p>
+
+<p>Look for a future post with many more details on how I setup Chef (likely incorrect) with more in-depth code and
+ configuration from the cookbooks.</p>
+
+]]></content>
+ </entry>
+
+ <entry>
<title type="html"><![CDATA[Scaling Your Clouds]]></title>
<link href="http://justcramer.com/2012/06/03/scaling-your-clouds"/>
<updated>2012-06-03T17:53:00-07:00</updated>
View
24 categories/heroku/index.html
@@ -72,6 +72,30 @@ <h1 class="archive-title">2012</h1>
<article class="archive">
+<h1 class="title"><a href="/2012/08/30/how-noops-works-for-sentry">Moving Sentry from Heroku to Hardware</a></h1>
+<div class="meta">
+ <span class="date">Aug 30</span>
+ <span class="tags">
+
+<div class="cat">
+
+ <a class='category' href='/categories/django/'>django</a>, <a class='category' href='/categories/heroku/'>heroku</a>, <a class='category' href='/categories/ops/'>ops</a>, <a class='category' href='/categories/sentry/'>sentry</a>
+
+</div>
+
+</span>
+
+ <span class="comments"><a href="/2012/08/30/how-noops-works-for-sentry#disqus_thread">Comments</a></span>
+
+</div>
+ </article>
+</section>
+
+<section class="archive">
+
+
+ <article class="archive">
+
<h1 class="title"><a href="/2012/06/03/scaling-your-clouds">Scaling Your Clouds</a></h1>
<div class="meta">
<span class="date">Jun 3</span>
View
2  categories/howto/atom.xml
@@ -4,7 +4,7 @@
<title><![CDATA[Category: howto | David Cramer's Blog]]></title>
<link href="http://justcramer.com/categories/howto/atom.xml" rel="self"/>
<link href="http://justcramer.com/"/>
- <updated>2012-08-30T22:21:30-07:00</updated>
+ <updated>2012-08-30T22:29:48-07:00</updated>
<id>http://justcramer.com/</id>
<author>
<name><![CDATA[David Cramer]]></name>
View
2  categories/ops/atom.xml
@@ -4,7 +4,7 @@
<title><![CDATA[Category: ops | David Cramer's Blog]]></title>
<link href="http://justcramer.com/categories/ops/atom.xml" rel="self"/>
<link href="http://justcramer.com/"/>
- <updated>2012-08-30T22:21:30-07:00</updated>
+ <updated>2012-08-30T22:29:48-07:00</updated>
<id>http://justcramer.com/</id>
<author>
<name><![CDATA[David Cramer]]></name>
View
2  categories/ops/index.html
@@ -79,7 +79,7 @@ <h1 class="title"><a href="/2012/08/30/how-noops-works-for-sentry">Moving Sentry
<div class="cat">
- <a class='category' href='/categories/django/'>django</a>, <a class='category' href='/categories/ops/'>ops</a>, <a class='category' href='/categories/sentry/'>sentry</a>
+ <a class='category' href='/categories/django/'>django</a>, <a class='category' href='/categories/heroku/'>heroku</a>, <a class='category' href='/categories/ops/'>ops</a>, <a class='category' href='/categories/sentry/'>sentry</a>
</div>
View
2  categories/osx/atom.xml
@@ -4,7 +4,7 @@
<title><![CDATA[Category: osx | David Cramer's Blog]]></title>
<link href="http://justcramer.com/categories/osx/atom.xml" rel="self"/>
<link href="http://justcramer.com/"/>
- <updated>2012-08-30T22:21:30-07:00</updated>
+ <updated>2012-08-30T22:29:48-07:00</updated>
<id>http://justcramer.com/</id>
<author>
<name><![CDATA[David Cramer]]></name>
View
2  categories/postgresql/atom.xml
@@ -4,7 +4,7 @@
<title><![CDATA[Category: postgresql | David Cramer's Blog]]></title>
<link href="http://justcramer.com/categories/postgresql/atom.xml" rel="self"/>
<link href="http://justcramer.com/"/>
- <updated>2012-08-30T22:21:30-07:00</updated>
+ <updated>2012-08-30T22:29:48-07:00</updated>
<id>http://justcramer.com/</id>
<author>
<name><![CDATA[David Cramer]]></name>
View
2  categories/python/atom.xml
@@ -4,7 +4,7 @@
<title><![CDATA[Category: python | David Cramer's Blog]]></title>
<link href="http://justcramer.com/categories/python/atom.xml" rel="self"/>
<link href="http://justcramer.com/"/>
- <updated>2012-08-30T22:21:30-07:00</updated>
+ <updated>2012-08-30T22:29:48-07:00</updated>
<id>http://justcramer.com/</id>
<author>
<name><![CDATA[David Cramer]]></name>
View
2  categories/sentry/atom.xml
@@ -4,7 +4,7 @@
<title><![CDATA[Category: sentry | David Cramer's Blog]]></title>
<link href="http://justcramer.com/categories/sentry/atom.xml" rel="self"/>
<link href="http://justcramer.com/"/>
- <updated>2012-08-30T22:21:30-07:00</updated>
+ <updated>2012-08-30T22:29:48-07:00</updated>
<id>http://justcramer.com/</id>
<author>
<name><![CDATA[David Cramer]]></name>
View
2  categories/sentry/index.html
@@ -79,7 +79,7 @@ <h1 class="title"><a href="/2012/08/30/how-noops-works-for-sentry">Moving Sentry
<div class="cat">
- <a class='category' href='/categories/django/'>django</a>, <a class='category' href='/categories/ops/'>ops</a>, <a class='category' href='/categories/sentry/'>sentry</a>
+ <a class='category' href='/categories/django/'>django</a>, <a class='category' href='/categories/heroku/'>heroku</a>, <a class='category' href='/categories/ops/'>ops</a>, <a class='category' href='/categories/sentry/'>sentry</a>
</div>
View
2  categories/solr/atom.xml
@@ -4,7 +4,7 @@
<title><![CDATA[Category: solr | David Cramer's Blog]]></title>
<link href="http://justcramer.com/categories/solr/atom.xml" rel="self"/>
<link href="http://justcramer.com/"/>
- <updated>2012-08-30T22:21:30-07:00</updated>
+ <updated>2012-08-30T22:29:48-07:00</updated>
<id>http://justcramer.com/</id>
<author>
<name><![CDATA[David Cramer]]></name>
View
2  disqus.xml
@@ -2,7 +2,7 @@
<feed xmlns="http://www.w3.org/2005/Atom">
<title>JustCramer</title>
<link href="http://justcramer.com/atom.xml" rel="self"/>
- <updated>2012-08-30T22:21:30-07:00</updated>
+ <updated>2012-08-30T22:29:48-07:00</updated>
<id>http://justcramer.com/</id>
View
2  django.xml
@@ -2,7 +2,7 @@
<feed xmlns="http://www.w3.org/2005/Atom">
<title>JustCramer</title>
<link href="http://justcramer.com/atom.xml" rel="self"/>
- <updated>2012-08-30T22:21:30-07:00</updated>
+ <updated>2012-08-30T22:29:48-07:00</updated>
<id>http://justcramer.com/</id>
View
2  feeds/disqus.xml
@@ -2,7 +2,7 @@
<feed xmlns="http://www.w3.org/2005/Atom">
<title>JustCramer</title>
<link href="http://justcramer.com/atom.xml" rel="self"/>
- <updated>2012-08-30T22:21:30-07:00</updated>
+ <updated>2012-08-30T22:29:48-07:00</updated>
<id>http://justcramer.com/</id>
View
2  feeds/django.xml
@@ -2,7 +2,7 @@
<feed xmlns="http://www.w3.org/2005/Atom">
<title>JustCramer</title>
<link href="http://justcramer.com/atom.xml" rel="self"/>
- <updated>2012-08-30T22:21:30-07:00</updated>
+ <updated>2012-08-30T22:29:48-07:00</updated>
<id>http://justcramer.com/</id>
View
2  index.html
@@ -238,7 +238,7 @@ <h1 class="title"><a href="/2012/08/30/how-noops-works-for-sentry">Moving Sentry
<div class="cat">
- <a class='category' href='/categories/django/'>django</a>, <a class='category' href='/categories/ops/'>ops</a>, <a class='category' href='/categories/sentry/'>sentry</a>
+ <a class='category' href='/categories/django/'>django</a>, <a class='category' href='/categories/heroku/'>heroku</a>, <a class='category' href='/categories/ops/'>ops</a>, <a class='category' href='/categories/sentry/'>sentry</a>
</div>
Please sign in to comment.
Something went wrong with that request. Please try again.