initialize 2

README

@@ -0,0 +1 @@
+cdn.debian.net surrogate checkerw/ GAE

app.yaml

@@ -0,0 +1,47 @@
+application: debiancdn
+version: 1
+runtime: python
+api_version: 1
+
+handlers:
+
+- url: /remote_api
+  script: $PYTHON_LIB/google/appengine/ext/remote_api/handler.py
+  login: admin
+
+- url: /managesurrogate
+  script: managesurrogate.py
+
+- url: /checksurrogate
+  script: checksurrogate.py
+  login: admin
+
+- url: /deletesurrogate
+  script: managesurrogate.py
+  login: admin
+
+- url: /removesurrogate
+  script: managesurrogate.py
+  login: admin
+
+- url: /addsurrogate
+  script: managesurrogate.py
+  login: admin
+
+- url: /.*surrogate
+  script: managesurrogate.py
+  login: admin
+
+- url: /lists/.*
+  script: lists.py
+  login: admin
+
+- url: /favicon.ico
+  static_files: static/favicon.ico
+  upload: static/favicon.ico
+
+- url: /json/.*
+  script: json.py
+
+- url: .*
+  script: main.py

checksurrogate.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import cgi
+import os
+import socket
+import datetime
+import urllib2
+from models import Surrogate
+
+from google.appengine.api import users
+from google.appengine.ext import webapp
+from google.appengine.ext.webapp.util import run_wsgi_app
+from google.appengine.ext import db
+from google.appengine.ext.webapp import template
+
+class CheckSurrogate(webapp.RequestHandler):
+    def get(self):
+        socket.timeout(1)
+
+        check_server_count = 0
+        check_period = 120
+        t1 = datetime.datetime.now()
+        message = ''
+        lm = ''
+        dm = ''
+        ttmp = datetime.datetime.now()
+        keika = ttmp - t1
+
+        surrogates = Surrogate.all().order('checkpref').order('time')
+
+        for surrogate in surrogates:
+            if check_server_count >= 70:
+                break
+            if surrogate.checkpref > 100:
+                continue
+
+            ttmp = datetime.datetime.now()
+            keika = ttmp - t1
+            if keika > datetime.timedelta(0,20):
+                break
+
+            if surrogate.checkpref:
+                surrogate.checkpref += surrogate.checkpref
+            else:
+                surrogate.checkpref = 0
+
+            if surrogate.tracefile:
+                True
+            else:
+                tracefile = 'ftp-master.debian.org'
+
+            if surrogate.alive == None or t1 > surrogate.time + datetime.timedelta(0,check_period):
+                check_server_count += 1
+                dm = 'go check'
+                k = surrogate.ip
+                req = urllib2.Request(url="http://" + k + '/debian/project/trace/' + tracefile)
+                req.add_header('User-Agent',"Debian-cdn-mirror-ping/1.2")
+                try:
+                    f = urllib2.urlopen(req)
+                    lm = f.info()['Last-Modified']
+                    message = f.read()
+                    surrogate.alive = True
+                    surrogate.checkpref = 0
+                    lmt = datetime.datetime.strptime(lm, "%a, %d %b %Y %H:%M:%S GMT")
+                    surrogate.lastModifiedTime = lmt
+                except urllib2.HTTPError, e:
+                    message += "%s is not working. (HTTP error)" % (k)
+                    surrogate.alive = False
+                    surrogate.checkpref += 1
+                except urllib2.URLError, e:
+                    message += "%s is not working. (URL error)" % (k)
+                    surrogate.alive = False
+                    surrogate.checkpref += 1
+                except:
+                    message += "%s is not working. " % (k)
+                    surrogate.alive = False
+                    surrogate.checkpref += 1
+
+                surrogate.put()
+            else:
+                dm = 'no check: ' + "%s is checked less than %d sec ago (surrogate.time %s, %s)" % (surrogate.ip,check_period,surrogate.time,t1)
+
+        if users.get_current_user():
+            url = users.create_logout_url(self.request.uri)
+            url_linktext = 'Logout'
+        else:
+            url = users.create_login_url(self.request.uri)
+            url_linktext = 'Login'
+
+        t2 = datetime.datetime.now()
+        keika = t2 - t1
+        template_values = {
+            'surrogates': surrogates,
+            'url': url,
+            'url_linktext': url_linktext,
+            'message': message,
+            'lm':lm,
+            'dm':dm,
+            'keika':keika,
+            }
+
+        path = os.path.join(os.path.dirname(__file__), 'managesurrogate.html')
+        self.response.out.write(template.render(path, template_values))
+
+
+application = webapp.WSGIApplication(
+  [('/checksurrogate', CheckSurrogate)],
+  debug=True)
+
+def main():
+  run_wsgi_app(application)
+
+if __name__ == "__main__":
+  main()

cron.yaml

@@ -0,0 +1,4 @@
+cron:
+- description: every min surrogate dead or alive checker
+  url: /checksurrogate
+  schedule: every 1 minutes

helptool.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+
+import re
+def ipFormatCheck(ip_str):
+   pattern = r"\b(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b"
+   if re.match(pattern, ip_str):
+      return True
+   else:
+      return False
+
+def countryFormatCheck(c_str):
+    pattern = r"\b\w\w\w\b"
+    if re.match(pattern, c_str):
+        return True
+    else:
+        return False

index.yaml

@@ -0,0 +1,66 @@
+indexes:
+
+# AUTOGENERATED
+
+# This index.yaml is automatically updated whenever the dev_appserver
+# detects that a new type of query is run.  If you want to manage the
+# index.yaml file manually, remove the above marker line (the line
+# saying "# AUTOGENERATED").  If you want to manage some indexes
+# manually, move them above the marker line.  The index.yaml file is
+# automatically uploaded to the admin console when you next deploy
+# your application using appcfg.py.
+
+- kind: Surrogate
+  properties:
+  - name: alive
+  - name: checkpref
+  - name: country
+
+- kind: Surrogate
+  properties:
+  - name: alive
+  - name: country
+
+- kind: Surrogate
+  properties:
+  - name: alive
+  - name: country
+  - name: hostname
+
+- kind: Surrogate
+  properties:
+  - name: alive=
+  - name: checkpref
+  - name: country
+
+- kind: Surrogate
+  properties:
+  - name: checkpref
+  - name: country
+
+- kind: Surrogate
+  properties:
+  - name: checkpref
+  - name: time
+
+- kind: Surrogate
+  properties:
+  - name: country
+  - name: checkpref
+
+- kind: Surrogate
+  properties:
+  - name: country
+  - name: checkpref
+    direction: desc
+
+- kind: Surrogate
+  properties:
+  - name: country
+  - name: hostname
+
+- kind: Surrogate
+  properties:
+  - name: preference=
+  - name: checkpref
+  - name: country

json.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import cgi
+import os
+from models import Surrogate
+import helptool
+
+from google.appengine.api import users
+from google.appengine.ext import webapp
+from google.appengine.ext.webapp.util import run_wsgi_app
+from google.appengine.ext import db
+from google.appengine.ext.webapp import template
+
+class PlainAlive(webapp.RequestHandler):
+  def get(self):
+    surrogates = Surrogate.all().order('country').order('hostname').filter('alive =',True)
+
+    template_values = {
+      'surrogates': surrogates
+      }
+
+    path = os.path.join(os.path.dirname(__file__), 'json/plain')
+    self.response.headers['Content-Type'] = "application/json"
+    self.response.out.write(template.render(path, template_values))
+
+class PlainAll(webapp.RequestHandler):
+  def get(self):
+    surrogates = Surrogate.all().order('country').order('hostname')
+
+    template_values = {
+      'surrogates': surrogates
+      }
+
+    path = os.path.join(os.path.dirname(__file__), 'json/plain')
+    self.response.headers['Content-Type'] = "application/json"
+    self.response.out.write(template.render(path, template_values))
+
+
+application = webapp.WSGIApplication(
+  [("/json/alive", PlainAlive),
+   ("/json/all", PlainAll)
+   ],
+  debug=True)
+
+def main():
+  run_wsgi_app(application)
+
+if __name__ == "__main__":
+  main()

json/plain

@@ -0,0 +1,19 @@
+[
+{% for su in surrogates %}
+{% if forloop.first %}
+{
+{% else %}
+,{
+{% endif %}
+"country":"{{su.country}}",
+"hostname":"{{su.hostname}}",
+"ip":"{{su.ip}}",
+"type":"{{su.type}}",
+"preference":{{su.preference}},
+"time":"{{su.time}}",
+"alive":"{{su.alive}}",
+"continent":"{{su.continent}}",
+"checkpref":{{su.checkpref}},
+"tracefile":"{{su.tracefile}}"}
+{% endfor %}
+]

lists.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import cgi
+import os
+from models import Surrogate
+import helptool
+
+from google.appengine.api import users
+from google.appengine.ext import webapp
+from google.appengine.ext.webapp.util import run_wsgi_app
+from google.appengine.ext import db
+from google.appengine.ext.webapp import template
+
+
+class AddrLists(webapp.RequestHandler):
+  def get(self):
+    surrogates = Surrogate.all().order('country').order('hostname').filter('alive =',True)
+
+    addr_db = {}
+    for surrogate in surrogates:
+        funi = surrogate
+
+    self.response.headers['Content-Type'] = "application"
+    self.response.out.write(dir(addr_db))
+
+
+application = webapp.WSGIApplication(
+  [('/lists/addr', AddrLists)],
+  debug=True)
+
+def main():
+  run_wsgi_app(application)
+
+if __name__ == "__main__":
+  main()

main.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+#
+# Copyright 2007 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+from google.appengine.ext import webapp
+from google.appengine.ext.webapp import util
+
+
+class MainHandler(webapp.RequestHandler):
+
+  def get(self):
+    self.response.out.write('Hello world!')
+
+
+def main():
+  application = webapp.WSGIApplication([('/', MainHandler)],
+                                       debug=True)
+  util.run_wsgi_app(application)
+
+
+if __name__ == '__main__':
+  main()