Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

adding preFeed

  • Loading branch information...
commit 581c9e43831c68e585fed2fbe38092e0267c5f55 1 parent a514d73
@pkra pkra authored
Showing with 96 additions and 1 deletion.
  1. +3 −0  app.yaml
  2. +36 −0 bulkloader.yaml
  3. +1 −1  cron.yaml
  4. +56 −0 main.py
View
3  app.yaml
@@ -43,6 +43,9 @@ handlers:
- url: /globaltaglistworker
script: main.py
login: admin
+- url: /prefeed
+ script: main.py
+ login: admin
- url: .*
script: main.py
View
36 bulkloader.yaml
@@ -98,3 +98,39 @@ transformers:
- property: taglist
external_name: taglist
import_transform: transform.none_if_empty(list)
+
+- kind: PreFeed
+ connector: csv
+ connector_options:
+ # TODO: Add connector options here--these are specific to each connector.
+ property_map:
+ - property: __key__
+ external_name: key
+ export_transform: transform.key_id_or_name_as_string
+
+ - property: comments_url
+ external_name: comments_url
+
+ - property: person
+ external_name: person
+ # Type: String Stats: 128 properties of this type in this kind.
+
+ - property: category
+ external_name: category
+ # Type: String Stats: 128 properties of this type in this kind.
+
+ - property: language
+ external_name: language
+ # Type: String Stats: 128 properties of this type in this kind.
+
+ - property: posts_url
+ external_name: posts_url
+ # Type: String Stats: 128 properties of this type in this kind.
+
+ - property: homepage
+ external_name: homepage
+
+ - property: title
+ external_name: title
+ # Type: String Stats: 128 properties of this type in this kind.
+
View
2  cron.yaml
@@ -1,7 +1,7 @@
cron:
- description: check for feed updates
url: /allworker
- schedule: every 6 hours
+ schedule: every 8 hours
- description: clean up datastore
url: /cleanupdatastore
schedule: every monday 02:00
View
56 main.py
@@ -377,6 +377,48 @@ class Comment(Entry):
#######################################
+### PreFeed: for extending the database by generating most of the information for the feedobjects
+#######################################
+
+
+class PreFeed(db.Model):
+ posts_url = db.LinkProperty()
+ comments_url = db.StringProperty()
+ category = db.StringProperty()
+ person = db.StringProperty()
+ language = db.StringProperty()
+ homepage = db.StringProperty()
+ title = db.StringProperty()
+
+ def generate_feedobject(self):
+ x = Feed()
+ try:
+ x.posts_url = self.posts_url
+ x.homepage = self.homepage
+ x.title = self.title
+ x.listtitle = self.title.lower()
+ x.person = self.person
+ x.category = self.category
+ x.taglist = []
+ x.language = self.language
+ x.priority = 1
+ x.favicon = ""
+ x.comments_url = self.comments_url
+ x.comments_day = 0
+ x.comments_week = 0
+ x.posts_week = 0
+ x.posts_month = 0
+ x.checksum_posts = '' # checksum of original rss-file
+ x.checksum_comments = '' # checksum of original rss-file
+ x.last_successful_posts_fetch_date = datetime.datetime(1970,1,1)
+ x.last_successful_comments_fetch_date = datetime.datetime(1970,1,1)
+ x.put()
+ except Exception, e: # TODO more exception catching: 'NoneType' error when feed is malformed not enough for bug tracking.
+ logging.warning("There was an error processing the Feedobject :" + str(e))
+
+
+
+#######################################
### Storing for caching
#######################################
@@ -571,6 +613,19 @@ def get(self):
x.put()
self.response.set_status(200)
+#################################
+### GENERATING FEEDOBJECTS FROM PREFEEDs
+#################################
+
+
+class PreFeedWorker(webapp.RequestHandler): ### runs at \prefeed with admin privileges -- make sure to delete prefeeds afterwards!
+ def get(self):
+ logging.info("PreFeedWorker")
+ for prefeed in PreFeed.all():
+ prefeed.generate_feedobject()
+ self.response.set_status(200)
+
+
#################################
### CLEANING UP THE DATASTORE // DELETING OLD ENTRIES
@@ -669,6 +724,7 @@ def main():
('/feedtaglistfetch', FeedTagListFetchWorker),
('/feedtaglistworker', FeedTagListWorker),
('/globaltaglistworker', GlobalTagListWorker),
+ ('/prefeed', PreFeedWorker),
('/reboot', RebootCommand),
('/feed_pure', FeedHandlerPure),
('/feed_applied', FeedHandlerApplied),
Please sign in to comment.
Something went wrong with that request. Please try again.