-
Notifications
You must be signed in to change notification settings - Fork 0
/
pavement.py
713 lines (580 loc) · 21 KB
/
pavement.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
# -*- coding: utf-8 -*-
#########################################################################
#
# Copyright (C) 2016 OSGeo
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
#########################################################################
import os
import re
import shutil
import sys
import time
import urllib
import urllib2
import zipfile
import glob
import fileinput
import yaml
from setuptools.command import easy_install
from urlparse import urlparse
from paver.easy import task, options, cmdopts, needs
from paver.easy import path, sh, info, call_task
from paver.easy import BuildFailure
try:
from geonode.settings import GEONODE_APPS
except:
# probably trying to run install_win_deps.
pass
try:
from paver.path import pushd
except ImportError:
from paver.easy import pushd
assert sys.version_info >= (2, 6), \
SystemError("GeoNode Build requires python 2.6 or better")
dev_config = None
with open("dev_config.yml", 'r') as f:
dev_config = yaml.load(f)
def grab(src, dest, name):
download = True
if not dest.exists():
print 'Downloading %s' % name
elif not zipfile.is_zipfile(dest):
print 'Downloading %s (corrupt file)' % name
else:
download = False
if download:
if str(src).startswith("file://"):
src2 = src[7:]
if not os.path.exists(src2):
print "Source location (%s) does not exist" % str(src2)
else:
print "Copying local file from %s" % str(src2)
shutil.copyfile(str(src2), str(dest))
else:
urllib.urlretrieve(str(src), str(dest))
@task
@cmdopts([
('geoserver=', 'g', 'The location of the geoserver build (.war file).'),
('jetty=', 'j', 'The location of the Jetty Runner (.jar file).'),
])
def setup_geoserver(options):
"""Prepare a testing instance of GeoServer."""
download_dir = path('downloaded')
if not download_dir.exists():
download_dir.makedirs()
geoserver_dir = path('geoserver')
geoserver_bin = download_dir / os.path.basename(dev_config['GEOSERVER_URL'])
jetty_runner = download_dir / os.path.basename(dev_config['JETTY_RUNNER_URL'])
grab(options.get('geoserver', dev_config['GEOSERVER_URL']), geoserver_bin, "geoserver binary")
grab(options.get('jetty', dev_config['JETTY_RUNNER_URL']), jetty_runner, "jetty runner")
if not geoserver_dir.exists():
geoserver_dir.makedirs()
webapp_dir = geoserver_dir / 'geoserver'
if not webapp_dir:
webapp_dir.makedirs()
print 'extracting geoserver'
z = zipfile.ZipFile(geoserver_bin, "r")
z.extractall(webapp_dir)
_install_data_dir()
def _install_data_dir():
target_data_dir = path('geoserver/data')
if target_data_dir.exists():
target_data_dir.rmtree()
original_data_dir = path('geoserver/geoserver/data')
justcopy(original_data_dir, target_data_dir)
config = path('geoserver/data/security/auth/geonodeAuthProvider/config.xml')
with open(config) as f:
xml = f.read()
m = re.search('baseUrl>([^<]+)', xml)
xml = xml[:m.start(1)] + "http://localhost:8000/" + xml[m.end(1):]
with open(config, 'w') as f:
f.write(xml)
@task
def static(options):
with pushd('geonode/static'):
sh('grunt production')
@task
@needs([
'setup_geoserver',
])
def setup(options):
"""Get dependencies and prepare a GeoNode development environment."""
info(('GeoNode development environment successfully set up.'
'If you have not set up an administrative account,'
' please do so now. Use "paver start" to start up the server.'))
def grab_winfiles(url, dest, packagename):
# Add headers
headers = {'User-Agent': 'Mozilla 5.10'}
request = urllib2.Request(url, None, headers)
response = urllib2.urlopen(request)
with open(dest, 'wb') as writefile:
writefile.write(response.read())
@task
def win_install_deps(options):
"""
Install all Windows Binary automatically
This can be removed as wheels become available for these packages
"""
download_dir = path('downloaded').abspath()
if not download_dir.exists():
download_dir.makedirs()
win_packages = {
# required by transifex-client
"Py2exe": dev_config['WINDOWS']['py2exe'],
"Nose": dev_config['WINDOWS']['nose'],
# the wheel 1.9.4 installs but pycsw wants 1.9.3, which fails to compile
# when pycsw bumps their pyproj to 1.9.4 this can be removed.
"PyProj": dev_config['WINDOWS']['pyproj']
}
failed = False
for package, url in win_packages.iteritems():
tempfile = download_dir / os.path.basename(url)
grab_winfiles(url, tempfile, package)
try:
easy_install.main([tempfile])
except Exception, e:
failed = True
print "install failed with error: ", e
os.remove(tempfile)
if failed and sys.maxsize > 2**32:
print "64bit architecture is not currently supported"
print "try finding the 64 binaries for py2exe, nose, and pyproj"
elif failed:
print "install failed for py2exe, nose, and/or pyproj"
else:
print "Windows dependencies now complete. Run pip install -e geonode --use-mirrors"
@cmdopts([
('version=', 'v', 'Legacy GeoNode version of the existing database.')
])
@task
def upgradedb(options):
"""
Add 'fake' data migrations for existing tables from legacy GeoNode versions
"""
version = options.get('version')
if version in ['1.1', '1.2']:
sh("python manage.py migrate maps 0001 --fake")
sh("python manage.py migrate avatar 0001 --fake")
elif version is None:
print "Please specify your GeoNode version"
else:
print "Upgrades from version %s are not yet supported." % version
@task
def sync(options):
"""
Run the syncdb and migrate management commands to create and migrate a DB
"""
try:
sh("python manage.py migrate auth --fake-initial")
except:
pass
sh("python manage.py syncdb --noinput")
# sh("python manage.py migrate --noinput")
sh("python manage.py loaddata sample_admin.json")
@task
def package(options):
"""
Creates a tarball to use for building the system elsewhere
"""
import tarfile
import geonode
version = geonode.get_version()
# Use GeoNode's version for the package name.
pkgname = 'GeoNode-%s-all' % version
# Create the output directory.
out_pkg = path(pkgname)
out_pkg_tar = path("%s.tar.gz" % pkgname)
# Create a distribution in zip format for the geonode python package.
dist_dir = path('dist')
dist_dir.rmtree()
sh('python setup.py sdist --formats=zip')
with pushd('package'):
# Delete old tar files in that directory
for f in glob.glob('GeoNode*.tar.gz'):
old_package = path(f)
if old_package != out_pkg_tar:
old_package.remove()
if out_pkg_tar.exists():
info('There is already a package for version %s' % version)
return
# Clean anything that is in the oupout package tree.
out_pkg.rmtree()
out_pkg.makedirs()
support_folder = path('support')
install_file = path('install.sh')
# And copy the default files from the package folder.
justcopy(support_folder, out_pkg / 'support')
justcopy(install_file, out_pkg)
geonode_dist = path('..') / 'dist' / 'GeoNode-%s.zip' % version
justcopy(geonode_dist, out_pkg)
# Create a tar file with all files in the output package folder.
tar = tarfile.open(out_pkg_tar, "w:gz")
for file in out_pkg.walkfiles():
tar.add(file)
# Add the README with the license and important links to documentation.
tar.add('README', arcname=('%s/README.rst' % out_pkg))
tar.close()
# Remove all the files in the temporary output package directory.
out_pkg.rmtree()
# Report the info about the new package.
info("%s created" % out_pkg_tar.abspath())
@task
@needs(['start_geoserver',
'sync',
'start_django'])
@cmdopts([
('bind=', 'b', 'Bind server to provided IP address and port number.'),
('java_path=', 'j', 'Full path to java install for Windows'),
('foreground', 'f', 'Do not run in background but in foreground')
], share_with=['start_django', 'start_geoserver'])
def start():
"""
Start GeoNode (Django, GeoServer & Client)
"""
info("GeoNode is now available.")
@task
def stop_django():
"""
Stop the GeoNode Django application
"""
kill('python', 'runserver')
@task
def stop_geoserver():
"""
Stop GeoServer
"""
kill('java', 'geoserver')
@task
def stop():
"""
Stop GeoNode
"""
# windows needs to stop the geoserver first b/c we can't tell which python is running, so we kill everything
stop_geoserver()
info("Stopping GeoNode ...")
stop_django()
@cmdopts([
('bind=', 'b', 'Bind server to provided IP address and port number.')
])
@task
def start_django():
"""
Start the GeoNode Django application
"""
bind = options.get('bind', '')
#foreground = '' if options.get('foreground', False) else '&'
#sh('python manage.py runserver %s %s' % (bind, foreground))
sh('python manage.py runserver 0.0.0.0:8000')# % (bind, foreground))
@cmdopts([
('java_path=', 'j', 'Full path to java install for Windows')
])
@task
def start_geoserver(options):
"""
Start GeoServer with GeoNode extensions
"""
from geonode.settings import OGC_SERVER
GEOSERVER_BASE_URL = OGC_SERVER['default']['LOCATION']
url = GEOSERVER_BASE_URL
if urlparse(GEOSERVER_BASE_URL).hostname != 'localhost':
print "Warning: OGC_SERVER['default']['LOCATION'] hostname is not equal to 'localhost'"
if not GEOSERVER_BASE_URL.endswith('/'):
print "Error: OGC_SERVER['default']['LOCATION'] does not end with a '/'"
sys.exit(1)
download_dir = path('downloaded').abspath()
jetty_runner = download_dir / os.path.basename(dev_config['JETTY_RUNNER_URL'])
data_dir = path('geoserver/data').abspath()
web_app = path('geoserver/geoserver').abspath()
log_file = path('geoserver/jetty.log').abspath()
config = path('scripts/misc/jetty-runner.xml').abspath()
jetty_port = urlparse(GEOSERVER_BASE_URL).port
# @todo - we should not have set workdir to the datadir but a bug in geoserver
# prevents geonode security from initializing correctly otherwise
with pushd(data_dir):
javapath = "java"
loggernullpath = os.devnull
# checking if our loggernullpath exists and if not, reset it to something manageable
if loggernullpath == "nul":
try:
open("../../downloaded/null.txt", 'w+').close()
except IOError, e:
print "Chances are that you have Geoserver currently running. You \
can either stop all servers with paver stop or start only \
the django application with paver start_django."
sys.exit(1)
loggernullpath = "../../downloaded/null.txt"
try:
sh(('java -version'))
except:
print "Java was not found in your path. Trying some other options: "
javapath_opt = None
if os.environ.get('JAVA_HOME', None):
print "Using the JAVA_HOME environment variable"
javapath_opt = os.path.join(os.path.abspath(os.environ['JAVA_HOME']), "bin", "java.exe")
elif options.get('java_path'):
javapath_opt = options.get('java_path')
else:
print "Paver cannot find java in the Windows Environment. \
Please provide the --java_path flag with your full path to \
java.exe e.g. --java_path=C:/path/to/java/bin/java.exe"
sys.exit(1)
# if there are spaces
javapath = 'START /B "" "' + javapath_opt + '"'
sh((
'%(javapath)s -Xmx512m -XX:MaxPermSize=256m'
' -DGEOSERVER_DATA_DIR=%(data_dir)s'
# workaround for JAI sealed jar issue and jetty classloader
' -Dorg.eclipse.jetty.server.webapp.parentLoaderPriority=true'
' -jar %(jetty_runner)s'
' --port %(jetty_port)i'
' --log %(log_file)s'
' %(config)s'
' > %(loggernullpath)s &' % locals()
))
info('Starting GeoServer on %s' % url)
# wait for GeoServer to start
started = waitfor(url)
info('The logs are available at %s' % log_file)
if not started:
# If applications did not start in time we will give the user a chance
# to inspect them and stop them manually.
info(('GeoServer never started properly or timed out.'
'It may still be running in the background.'))
sys.exit(1)
@task
def test(options):
"""
Run GeoNode's Unit Test Suite
"""
sh("python manage.py test %s.tests --noinput" % '.tests '.join(GEONODE_APPS))
@task
def test_javascript(options):
with pushd('geonode/static/geonode'):
sh('./run-tests.sh')
@task
@cmdopts([
('name=', 'n', 'Run specific tests.')
])
def test_integration(options):
"""
Run GeoNode's Integration test suite against the external apps
"""
_reset()
# Start GeoServer
call_task('start_geoserver')
info("GeoNode is now available, running the tests now.")
name = options.get('name', 'geonode.tests.integration')
success = False
try:
if name == 'geonode.tests.csw':
call_task('start')
sh('sleep 30')
call_task('setup_data')
sh(('python manage.py test %s'
' --noinput --liveserver=localhost:8000' % name))
except BuildFailure, e:
info('Tests failed! %s' % str(e))
else:
success = True
finally:
# don't use call task here - it won't run since it already has
stop()
_reset()
if not success:
sys.exit(1)
@task
def run_tests():
"""
Executes the entire test suite.
"""
sh('python manage.py test geonode.tests.smoke')
call_task('test')
call_task('test_integration')
call_task('test_integration', options={'name': 'geonode.tests.csw'})
sh('flake8 geonode')
@task
@needs(['stop'])
def reset():
"""
Reset a development environment (Database, GeoServer & Catalogue)
"""
_reset()
def _reset():
sh("rm -rf geonode/development.db")
sh("rm -rf geonode/uploaded/*")
_install_data_dir()
@needs(['reset'])
def reset_hard():
"""
Reset a development environment (Database, GeoServer & Catalogue)
"""
sh("git clean -dxf")
@task
@cmdopts([
('type=', 't', 'Import specific data type ("vector", "raster", "time")'),
])
def setup_data():
"""
Import sample data (from gisdata package) into GeoNode
"""
import gisdata
ctype = options.get('type', None)
data_dir = gisdata.GOOD_DATA
if ctype in ['vector', 'raster', 'time']:
data_dir = os.path.join(gisdata.GOOD_DATA, ctype)
sh("python manage.py importlayers %s -v2" % data_dir)
@needs(['package'])
@cmdopts([
('key=', 'k', 'The GPG key to sign the package'),
('ppa=', 'p', 'PPA this package should be published to.'),
])
def deb(options):
"""
Creates debian packages.
Example uses:
paver deb
paver deb -k 12345
paver deb -k 12345 -p geonode/testing
"""
key = options.get('key', None)
ppa = options.get('ppa', None)
version, simple_version = versions()
info('Creating package for GeoNode version %s' % version)
# Get rid of any uncommitted changes to debian/changelog
info('Getting rid of any uncommitted changes in debian/changelog')
sh('git checkout package/debian/changelog')
# Workaround for git-dch bug
# http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=594580
sh('ln -s %s %s' % (os.path.realpath('.git'), os.path.realpath('package')))
with pushd('package'):
# Install requirements
# sh('sudo apt-get -y install debhelper devscripts git-buildpackage')
sh(('git-dch --spawn-editor=snapshot --git-author --new-version=%s'
' --id-length=6 --ignore-branch --release' % (simple_version)))
deb_changelog = path('debian') / 'changelog'
for line in fileinput.input([deb_changelog], inplace=True):
print line.replace("urgency=medium", "urgency=high"),
# Revert workaround for git-dhc bug
sh('rm -rf .git')
if key is None and ppa is None:
# A local installable package
sh('debuild -uc -us -A')
elif key is None and ppa is not None:
# A sources package, signed by daemon
sh('debuild -S')
elif key is not None and ppa is None:
# A signed installable package
sh('debuild -k%s -A' % key)
elif key is not None and ppa is not None:
# A signed, source package
sh('debuild -k%s -S' % key)
if ppa is not None:
sh('dput ppa:%s geonode_%s_source.changes' % (ppa, simple_version))
@task
def publish():
if 'GPG_KEY_GEONODE' in os.environ:
key = os.environ['GPG_KEY_GEONODE']
else:
print "You need to set the GPG_KEY_GEONODE environment variable"
return
call_task('deb', options={
'key': key,
'ppa': 'geonode/testing',
})
version, simple_version = versions()
sh('git add package/debian/changelog')
sh('git commit -m "Updated changelog for version %s"' % version)
sh('git tag %s' % version)
sh('git push origin %s' % version)
sh('git tag debian/%s' % simple_version)
sh('git push origin debian/%s' % simple_version)
sh('git push origin master')
sh('python setup.py sdist upload')
def versions():
import geonode
from geonode.version import get_git_changeset
raw_version = geonode.__version__
version = geonode.get_version()
timestamp = get_git_changeset()
major, minor, revision, stage, edition = raw_version
branch = 'dev'
if stage == 'final':
stage = 'thefinal'
if stage == 'alpha' and edition == 0:
tail = '%s%s' % (branch, timestamp)
else:
tail = '%s%s' % (stage, edition)
simple_version = '%s.%s.%s+%s' % (major, minor, revision, tail)
return version, simple_version
def kill(arg1, arg2):
"""Stops a proces that contains arg1 and is filtered by arg2
"""
from subprocess import Popen, PIPE
# Wait until ready
t0 = time.time()
# Wait no more than these many seconds
time_out = 30
running = True
while running and time.time() - t0 < time_out:
if os.name == 'nt':
p = Popen('tasklist | find "%s"' % arg1, shell=True,
stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=False)
else:
p = Popen('ps aux | grep %s' % arg1, shell=True,
stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
lines = p.stdout.readlines()
running = False
for line in lines:
# this kills all java.exe and python including self in windows
if ('%s' % arg2 in line) or (os.name == 'nt' and '%s' % arg1 in line):
running = True
# Get pid
fields = line.strip().split()
info('Stopping %s (process number %s)' % (arg1, fields[1]))
if os.name == 'nt':
kill = 'taskkill /F /PID "%s"' % fields[1]
else:
kill = 'kill -9 %s 2> /dev/null' % fields[1]
os.system(kill)
# Give it a little more time
time.sleep(1)
else:
pass
if running:
raise Exception('Could not stop %s: '
'Running processes are\n%s'
% (arg1, '\n'.join([l.strip() for l in lines])))
def waitfor(url, timeout=300):
started = False
for a in xrange(timeout):
try:
resp = urllib.urlopen(url)
except IOError:
pass
else:
if resp.getcode() == 200:
started = True
break
time.sleep(1)
return started
def justcopy(origin, target):
if os.path.isdir(origin):
shutil.rmtree(target, ignore_errors=True)
shutil.copytree(origin, target)
elif os.path.isfile(origin):
if not os.path.exists(target):
os.makedirs(target)
shutil.copy(origin, target)