Permalink
Browse files

Removing unused original pages from S3 when stored on node server.

  • Loading branch information...
1 parent e4fa40b commit 9f896d0493217346d67f144b02e6ce8bb1f40252 @samuelclay committed Dec 18, 2012
Showing with 50 additions and 13 deletions.
  1. +11 −1 apps/rss_feeds/page_importer.py
  2. +10 −0 config/supervisor_node_original.conf
  3. +29 −12 fabfile.py
@@ -180,6 +180,8 @@ def save_page(self, html):
if settings.BACKED_BY_AWS.get('pages_on_node'):
saved = self.save_page_node(html)
+ if saved and self.feed.s3_page:
+ self.delete_page_s3()
if settings.BACKED_BY_AWS.get('pages_on_s3') and not saved:
saved = self.save_page_s3(html)
@@ -224,4 +226,12 @@ def save_page_s3(self, html):
self.feed.s3_page = True
self.feed.save()
- return True
+ return True
+
+ def delete_page_s3(self):
+ k = Key(settings.S3_PAGES_BUCKET)
+ k.key = self.feed.s3_pages_key
+ k.delete()
+
+ self.feed.s3_page = False
+ self.feed.save()
@@ -0,0 +1,10 @@
+[program:node_original_page]
+command=node original_page.js
+directory=/srv/newsblur/node
+user=sclay
+autostart=true
+autorestart=true
+#redirect_stderr=True
+priority=991
+stopsignal=HUP
+stdout_logfile = /srv/newsblur/logs/original_page.log
View
@@ -629,22 +629,31 @@ def maintenance_off():
# ==============
def setup_db_firewall():
+ ports = [
+ 5432, # PostgreSQL
+ 27017, # MongoDB
+ 6379, # Redis
+ 11211, # Memcached
+ 3060, # Node original page server
+ ]
sudo('ufw default deny')
sudo('ufw allow ssh')
sudo('ufw allow 80')
- sudo('ufw allow from 199.15.248.0/21 to any port 5432 ') # PostgreSQL
- sudo('ufw allow from 199.15.248.0/21 to any port 27017') # MongoDB
- sudo('ufw allow from 199.15.248.0/21 to any port 28017') # MongoDB web
- sudo('ufw allow from 199.15.248.0/21 to any port 6379 ') # Redis
- sudo('ufw allow from 199.15.248.0/21 to any port 11211 ') # Memcached
+ for port in ports:
+ sudo('ufw allow from 199.15.248.0/21 to any port %s ' % port)
# EC2
- sudo('ufw allow proto tcp from 54.242.38.48 to any port 5432,27017,6379,11211')
- sudo('ufw allow proto tcp from 184.72.214.147 to any port 5432,27017,6379,11211')
- sudo('ufw allow proto tcp from 107.20.103.16 to any port 5432,27017,6379,11211')
- sudo('ufw allow proto tcp from 50.17.12.16 to any port 5432,27017,6379,11211')
- sudo('ufw allow proto tcp from 184.73.2.61 to any port 5432,27017,6379,11211')
- sudo('ufw allow proto tcp from 54.242.34.138 to any port 5432,27017,6379,11211')
+ for host in env.roledefs['ec2task']:
+ sudo('ufw allow proto tcp from %s to any port %s' % (
+ host,
+ ','.join(ports)
+ ))
+ # sudo('ufw allow proto tcp from 54.242.38.48 to any port 5432,27017,6379,11211')
+ # sudo('ufw allow proto tcp from 184.72.214.147 to any port 5432,27017,6379,11211')
+ # sudo('ufw allow proto tcp from 107.20.103.16 to any port 5432,27017,6379,11211')
+ # sudo('ufw allow proto tcp from 50.17.12.16 to any port 5432,27017,6379,11211')
+ # sudo('ufw allow proto tcp from 184.73.2.61 to any port 5432,27017,6379,11211')
+ # sudo('ufw allow proto tcp from 54.242.34.138 to any port 5432,27017,6379,11211')
sudo('ufw --force enable')
def setup_db_motd():
@@ -744,7 +753,15 @@ def setup_db_mdadm():
sudo("mdadm --examine --scan | sudo tee -a /etc/mdadm/mdadm.conf")
sudo("echo '/dev/md0 /srv/db xfs rw,nobarrier,noatime,nodiratime,noauto 0 0' | sudo tee -a /etc/fstab")
sudo("sudo update-initramfs -u -v -k `uname -r`")
-
+
+def setup_original_page_server():
+ setup_node()
+ sudo('rm -fr /etc/supervisor/conf.d/node.conf')
+ put('config/supervisor_node_original.conf',
+ '/etc/supervisor/conf.d/node_original.conf', use_sudo=True)
+ sudo('supervisorctl reread')
+ sudo('supervisorctl reload')
+
# ================
# = Setup - Task =
# ================

0 comments on commit 9f896d0

Please sign in to comment.