Added --overwrite-output (-O) option

scrapy/commands/crawl.py

@@ -20,6 +20,8 @@ def add_options(self, parser):
                           help="set spider argument (may be repeated)")
         parser.add_option("-o", "--output", metavar="FILE",
                           help="dump scraped items into FILE (use - for stdout)")
+        parser.add_option("-O", "--overwrite-output", metavar="FILE",
+                          help="overwrite scraped items into FILE")
         parser.add_option("-t", "--output-format", metavar="FORMAT",
                           help="format to use for dumping items with -o")
 
@@ -29,6 +31,11 @@ def process_options(self, args, opts):
             opts.spargs = arglist_to_dict(opts.spargs)
         except ValueError:
             raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
+        if opts.overwrite_output:
+            if opts.output:
+                raise UsageError("Please use only one of --output and --overwrite-output")
+            opts.output = opts.overwrite_output
+            self.settings.overrides['FEED_URI_OVERWRITE'] = True
         if opts.output:
             if opts.output == '-':
                 self.settings.overrides['FEED_URI'] = 'stdout:'

scrapy/commands/runspider.py

@@ -43,6 +43,8 @@ def add_options(self, parser):
                           help="set spider argument (may be repeated)")
         parser.add_option("-o", "--output", metavar="FILE",
                           help="dump scraped items into FILE (use - for stdout)")
+        parser.add_option("-O", "--overwrite-output", metavar="FILE",
+                          help="overwrite scraped items into FILE")
         parser.add_option("-t", "--output-format", metavar="FORMAT",
                           help="format to use for dumping items with -o")
 
@@ -52,6 +54,11 @@ def process_options(self, args, opts):
             opts.spargs = arglist_to_dict(opts.spargs)
         except ValueError:
             raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
+        if opts.overwrite_output:
+            if opts.output:
+                raise UsageError("Please use only one of --output and --overwrite-output")
+            opts.output = opts.overwrite_output
+            self.settings.overrides['FEED_URI_OVERWRITE'] = True
         if opts.output:
             if opts.output == '-':
                 self.settings.overrides['FEED_URI'] = 'stdout:'

scrapy/contrib/feedexport.py

@@ -67,13 +67,15 @@ class FileFeedStorage(object):
     implements(IFeedStorage)
 
     def __init__(self, uri):
+        from scrapy.conf import settings
         self.path = file_uri_to_path(uri)
+        self.overwrite = settings['FEED_URI_OVERWRITE']
 
     def open(self, spider):
         dirname = os.path.dirname(self.path)
         if dirname and not os.path.exists(dirname):
             os.makedirs(dirname)
-        return open(self.path, 'ab')
+        return open(self.path, 'wb' if self.overwrite else 'ab')
 
     def store(self, file):
         file.close()

scrapy/settings/default_settings.py

@@ -119,6 +119,7 @@
 }
 
 FEED_URI = None
+FEED_URI_OVERWRITE = False
 FEED_URI_PARAMS = None  # a function to extend uri arguments
 FEED_FORMAT = 'jsonlines'
 FEED_STORE_EMPTY = False