Skip to content
This repository

Serverside template #4

Closed
wants to merge 4 commits into from

1 participant

Jordi Llonch
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
This page is out of date. Refresh to see the latest.
1  MANIFEST.in
@@ -7,3 +7,4 @@ recursive-include scrapyd/tests *.egg
7 7
 recursive-include docs *
8 8
 prune docs/build
9 9
 recursive-include extras *
  10
+recursive-include htdocs *
2  scrapyd/default_scrapyd.conf
@@ -2,6 +2,7 @@
2 2
 eggs_dir    = eggs
3 3
 logs_dir    = logs
4 4
 items_dir   = items
  5
+htdocs_dir  = 
5 6
 jobs_to_keep = 5
6 7
 dbs_dir     = dbs
7 8
 max_proc    = 0
@@ -23,3 +24,4 @@ listspiders.json  = scrapyd.webservice.ListSpiders
23 24
 delproject.json   = scrapyd.webservice.DeleteProject
24 25
 delversion.json   = scrapyd.webservice.DeleteVersion
25 26
 listjobs.json     = scrapyd.webservice.ListJobs
  27
+status.json       = scrapyd.webservice.Status
111  scrapyd/htdocs/index.html
... ...
@@ -0,0 +1,111 @@
  1
+<!DOCTYPE html>
  2
+<html lang="en" ng-app="app">
  3
+  <head>
  4
+    <meta charset="utf-8">
  5
+    <title>Scrapyd</title>
  6
+
  7
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
  8
+    <meta name="description" content="Scrapy server">
  9
+    <meta name="author" content="Jordi Llonch <llonchj@gmail.com>">
  10
+
  11
+    <!-- Le styles -->
  12
+    <style>
  13
+      body {
  14
+        padding-top: 60px; /* 60px to make the container go all the way to the bottom of the topbar */
  15
+      }
  16
+    </style>
  17
+    <link href="//netdna.bootstrapcdn.com/twitter-bootstrap/2.2.2/css/bootstrap-combined.min.css" rel="stylesheet">
  18
+
  19
+    <!-- HTML5 shim, for IE6-8 support of HTML5 elements -->
  20
+    <!--[if lt IE 9]>
  21
+      <script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
  22
+    <![endif]-->
  23
+
  24
+    <script src="//ajax.googleapis.com/ajax/libs/jquery/1.9.0/jquery.min.js"></script>
  25
+    <script src="//ajax.googleapis.com/ajax/libs/angularjs/1.0.4/angular.min.js"></script>
  26
+    <script src="//ajax.googleapis.com/ajax/libs/angularjs/1.0.4/angular-resource.min.js"></script>
  27
+    <script src="//netdna.bootstrapcdn.com/twitter-bootstrap/2.2.2/js/bootstrap.min.js"></script>
  28
+
  29
+    <script src="/js/scrapyd.js"></script>
  30
+
  31
+    <!-- Fav and touch icons -->
  32
+    <link rel="shortcut icon" href="/ico/favicon.png">
  33
+  </head>
  34
+
  35
+  <body ng-controller="AppCtrl">
  36
+    <div class="navbar navbar-inverse navbar-fixed-top">
  37
+      <div class="navbar-inner">
  38
+        <div class="container">
  39
+          <a class="brand" href="/">[[appname]]</a>
  40
+          <ul class="nav">
  41
+            <li class="dropdown">
  42
+              <a class="dropdown-toggle" data-toggle="dropdown" href="#">
  43
+                Project
  44
+                <span class="caret"></span>
  45
+              </a>
  46
+              <ul class="dropdown-menu">
  47
+                <li ng-repeat="project in projects"><a href="#" id="id_{{project}}">{{project}}</a></li>
  48
+              </ul>
  49
+            </li>
  50
+
  51
+            <li class="dropdown">
  52
+              <a class="dropdown-toggle" data-toggle="dropdown" href="#">
  53
+                Spider
  54
+                <span class="caret"></span>
  55
+              </a>
  56
+              <ul class="dropdown-menu">
  57
+                <li ng-repeat="spider in spiders"><a href="#" id="id_{{spider}}">{{spider}}</a></li>
  58
+              </ul>
  59
+            </li>
  60
+
  61
+            <li><a href="http://doc.scrapy.org/en/latest/topics/scrapyd.html" target="_new">Documentation</a></li>
  62
+          </ul>
  63
+        </div>
  64
+      </div>
  65
+    </div>
  66
+
  67
+    <div class="container">
  68
+
  69
+      <div class="modal hide" id="myModal" tabindex="-1" role="dialog" aria-labelledby="myModalLabel" aria-hidden="true">
  70
+        <div class="modal-header">
  71
+          <button type="button" class="close" data-dismiss="modal" aria-hidden="true">×</button>
  72
+          <h3 id="myModalLabel">Start crawler {{spider}}</h3>
  73
+        </div>
  74
+        <div class="modal-body">
  75
+          <h4>Settings</h4>
  76
+          <textarea ></textarea>
  77
+          <h4>Arguments</h4>
  78
+          <textarea></textarea>
  79
+        </div>
  80
+        <div class="modal-footer">
  81
+          <button class="btn" data-dismiss="modal" aria-hidden="true">Cancel</button>
  82
+          <button class="btn btn-primary">Schedule</button>
  83
+        </div>
  84
+      </div>
  85
+      
  86
+
  87
+      <a href="#myModal" role="button" class="btn" data-toggle="modal">Schedule</a>
  88
+      
  89
+      <button ng-click="refresh()">refresh</button>
  90
+
  91
+
  92
+
  93
+      <table class="table table-bordered">
  94
+        <th>Project</th><th>Spider</th><th>Job</th><th>Status</th><th>Time</th>
  95
+          <tr ng-repeat="task in tasks">
  96
+            <td>{{task.project}}</td>
  97
+            <td>{{task.spider}}</td>
  98
+            <td>
  99
+              <span ng:hide="task.status == 'pending'" >
  100
+                <a class="icon-book" title="Log" href="/logs/{{task.project}}/{{task.spider}}/{{task.job}}.log"></a>
  101
+                <a class="icon-file" title="Items" href="/items/{{task.project}}/{{task.spider}}/{{task.job}}.jl"></a>
  102
+              </span>
  103
+                {{task.job}}</td>
  104
+            <td>{{task.status}}</td>
  105
+            <td>{{task.elapsed}}</td>
  106
+          </tr>
  107
+      </table>
  108
+    </div> <!-- /container -->
  109
+
  110
+  </body>
  111
+</html>
19  scrapyd/htdocs/js/scrapyd.js
... ...
@@ -0,0 +1,19 @@
  1
+/*
  2
+Scrapyd Javascript
  3
+*/
  4
+
  5
+var app = angular.module('app', ['ngResource']);
  6
+
  7
+var AppCtrl = function($scope, $http) {
  8
+
  9
+  $scope.refresh = function() {
  10
+    $http.get('/status.json')
  11
+      .success(function(data) {
  12
+      $scope.projects = data.projects;
  13
+    });
  14
+  }
  15
+
  16
+  $scope.refresh();
  17
+}
  18
+
  19
+app.controller('AppCtrl', AppCtrl);
11  scrapyd/webservice.py
@@ -120,3 +120,14 @@ def render_POST(self, txrequest):
120 120
         version = txrequest.args['version'][0]
121 121
         self._delete_version(project, version)
122 122
         return {"status": "ok"}
  123
+
  124
+class Status(WsResource):
  125
+
  126
+    def render_GET(self, txrequest):
  127
+        projects = {}
  128
+        for project in self.root.scheduler.list_projects():
  129
+            spiders = get_spider_list(project, runner=self.root.runner)
  130
+            versions = self.root.eggstorage.list(project)
  131
+            projects[project] = {"spiders": spiders, "versions": versions}
  132
+        return {"status": "ok", "projects":projects}
  133
+
60  scrapyd/website.py
... ...
@@ -1,8 +1,13 @@
  1
+import posixpath
  2
+import pkg_resources
  3
+
1 4
 from datetime import datetime
2 5
 
3 6
 from twisted.web import resource, static
4 7
 from twisted.application.service import IServiceCollection
5 8
 
  9
+from jinja2 import Template, Environment, FileSystemLoader
  10
+
6 11
 from scrapy.utils.misc import load_object
7 12
 
8 13
 from .interfaces import IPoller, IEggStorage, ISpiderScheduler
@@ -13,10 +18,18 @@ def __init__(self, config, app):
13 18
         resource.Resource.__init__(self)
14 19
         self.debug = config.getboolean('debug', False)
15 20
         self.runner = config.get('runner')
  21
+
  22
+        self.htdocsdir = (config.get('htdocs_dir') or 
  23
+            pkg_resources.resource_filename(__name__, 'htdocs'))
  24
+
  25
+        self.environ = Environment(loader=FileSystemLoader(self.htdocsdir))
  26
+        self.environ.variable_start_string = '[['
  27
+        self.environ.variable_end_string = ']]'
  28
+
16 29
         logsdir = config.get('logs_dir')
17 30
         itemsdir = config.get('items_dir')
18 31
         self.app = app
19  
-        self.putChild('', Home(self))
  32
+
20 33
         self.putChild('logs', static.File(logsdir, 'text/plain'))
21 34
         self.putChild('items', static.File(itemsdir, 'text/plain'))
22 35
         self.putChild('jobs', Jobs(self))
@@ -26,6 +39,10 @@ def __init__(self, config, app):
26 39
           self.putChild(servName, servCls(self))
27 40
         self.update_projects()
28 41
 
  42
+    def getChild(self, name, request):
  43
+        return (Renderer(self, name) if name=="" or name.endswith(".html") else 
  44
+                static.File(posixpath.join(self.htdocsdir, name)))
  45
+
29 46
     def update_projects(self):
30 47
         self.poller.update_projects()
31 48
         self.scheduler.update_projects()
@@ -47,43 +64,24 @@ def eggstorage(self):
47 64
     def poller(self):
48 65
         return self.app.getComponent(IPoller)
49 66
 
  67
+class Renderer(resource.Resource):
50 68
 
51  
-class Home(resource.Resource):
52  
-
53  
-    def __init__(self, root):
  69
+    def __init__(self, root, name, document_root='index.html'):
54 70
         resource.Resource.__init__(self)
55 71
         self.root = root
  72
+        self.name = name or document_root
56 73
 
57 74
     def render_GET(self, txrequest):
58  
-        vars = {
59  
-            'projects': ', '.join(self.root.scheduler.list_projects()),
  75
+        ctx = {
  76
+            'appname': "Scrapy",
  77
+            'projects': self.root.scheduler.list_projects(),
  78
+            'queues': self.root.poller.queues,
  79
+            'launcher': self.root.launcher,
60 80
         }
61  
-        return """
62  
-<html>
63  
-<head><title>Scrapyd</title></head>
64  
-<body>
65  
-<h1>Scrapyd</h1>
66  
-<p>Available projects: <b>%(projects)s</b></p>
67  
-<ul>
68  
-<li><a href="/jobs">Jobs</a></li>
69  
-<li><a href="/items/">Items</li>
70  
-<li><a href="/logs/">Logs</li>
71  
-<li><a href="http://doc.scrapy.org/en/latest/topics/scrapyd.html">Documentation</a></li>
72  
-</ul>
73  
-
74  
-<h2>How to schedule a spider?</h2>
75  
-
76  
-<p>To schedule a spider you need to use the API (this web UI is only for
77  
-monitoring)</p>
78  
-
79  
-<p>Example using <a href="http://curl.haxx.se/">curl</a>:</p>
80  
-<p><code>curl http://localhost:6800/schedule.json -d project=default -d spider=somespider</code></p>
81  
-
82  
-<p>For more information about the API, see the <a href="http://doc.scrapy.org/en/latest/topics/scrapyd.html">Scrapyd documentation</a></p>
83  
-</body>
84  
-</html>
85  
-""" % vars
86 81
 
  82
+        template = self.root.environ.get_template(self.name)
  83
+        response = template.render(**ctx)
  84
+        return response.encode("utf-8")
87 85
 
88 86
 class Jobs(resource.Resource):
89 87
 
2  setup.py
@@ -127,6 +127,6 @@ def is_not_module(filename):
127 127
 except ImportError:
128 128
     from distutils.core import setup
129 129
 else:
130  
-    setup_args['install_requires'] = ['Twisted>=8.0', 'Scrapy>=0.17']
  130
+    setup_args['install_requires'] = ['Twisted>=8.0', 'Scrapy>=0.17', 'Jinja2']
131 131
 
132 132
 setup(**setup_args)
Commit_comment_tip

Tip: You can add notes to lines in a file. Hover to the left of a line to make a note

Something went wrong with that request. Please try again.