forked from jjlee/mechanize
-
Notifications
You must be signed in to change notification settings - Fork 0
/
release.py
1086 lines (949 loc) · 43.3 KB
/
release.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""%prog RELEASE_AREA [action ...]
Perform needed actions to release mechanize, doing the work in directory
RELEASE_AREA.
If no actions are given, print the tree of actions and do nothing.
This is only intended to work on Unix (unlike mechanize itself). Some of it
only works on Ubuntu 10.04 (lucid).
Warning:
* Many actions do rm -rf on RELEASE_AREA or subdirectories of RELEASE_AREA.
* The install_deps action installs some debian packages system-wide. The
clean action doesn't uninstall them.
* The install_deps action adds a PPA.
* The install_deps action downloads and installs software to RELEASE_AREA.
The clean action uninstalls (by rm -rf).
"""
# This script depends on the code from this git repository:
# git://github.com/jjlee/mechanize-build-tools.git
# TODO
# * Tag mechanize-build-tools repository when releasing so that builds are
# reproducible
# * 0install package?
# * test in a Windows VM
import glob
import optparse
import os
import re
import shutil
import subprocess
import sys
import tempfile
import time
import unittest
# Stop the test runner from reporting import failure if these modules aren't
# available or not running under Python >= 2.6. AttributeError occurs if run
# with Python < 2.6, due to lack of collections.namedtuple
try:
import action_tree
import build_log
import cmd_env
import buildtools.release as release
except (ImportError, AttributeError):
# fake module
class action_tree(object):
@staticmethod
def action_node(func):
return func
# based on Mark Seaborn's plash build-tools (action_tree) and Cmed's in-chroot
# (cmd_env) -- which is also Mark's idea
class WrongVersionError(Exception):
def __init__(self, version):
Exception.__init__(self, version)
self.version = version
def __str__(self):
return str(self.version)
class MissingVersionError(Exception):
def __init__(self, path, release_version):
Exception.__init__(self, path, release_version)
self.path = path
self.release_version = release_version
def __str__(self):
return ("Release version string not found in %s: should be %s" %
(self.path, self.release_version))
class CSSValidationError(Exception):
def __init__(self, path, details):
Exception.__init__(self, path, details)
self.path = path
self.details = details
def __str__(self):
return ("CSS validation of %s failed:\n%s" %
(self.path, self.details))
def run_performance_tests(path):
# TODO: use a better/standard test runner
sys.path.insert(0, os.path.join(path, "test"))
test_runner = unittest.TextTestRunner(verbosity=1)
test_loader = unittest.defaultTestLoader
modules = []
for module_name in ["test_performance"]:
module = __import__(module_name)
for part in module_name.split('.')[1:]:
module = getattr(module, part)
modules.append(module)
suite = unittest.TestSuite()
for module in modules:
test = test_loader.loadTestsFromModule(module)
suite.addTest(test)
result = test_runner.run(test)
return result
def clean_environ_env(env):
return cmd_env.PrefixCmdEnv(
["sh", "-c", 'env -i HOME="$HOME" PATH="$PATH" "$@"',
"clean_environ_env"], env)
def check_version_equals(env, version, python):
try:
output = release.get_cmd_stdout(
env,
[python, "-c",
"import mechanize; print mechanize.__version__"],
stderr=subprocess.PIPE)
except cmd_env.CommandFailedError:
raise WrongVersionError(None)
else:
version_tuple_string = output.strip()
assert len(version.tuple) == 6, len(version.tuple)
if not(version_tuple_string == str(version.tuple) or
version_tuple_string == str(version.tuple[:-1])):
raise WrongVersionError(version_tuple_string)
def check_not_installed(env, python):
bogus_version = release.parse_version("0.0.0")
try:
check_version_equals(env, bogus_version, python)
except WrongVersionError, exc:
if exc.version is not None:
raise
else:
raise WrongVersionError(bogus_version)
class EasyInstallTester(object):
def __init__(self, env, install_dir, project_name,
test_cmd, expected_version,
easy_install_cmd=("easy_install",),
python="python"):
self._env = env
self._install_dir = install_dir
self._project_name = project_name
self._test_cmd = test_cmd
self._expected_version = expected_version
self._easy_install_cmd = list(easy_install_cmd)
self._python = python
self._install_dir_on_pythonpath = cmd_env.set_environ_vars_env(
[("PYTHONPATH", self._install_dir)], env)
def easy_install(self, log):
release.clean_dir(self._env, self._install_dir)
check_not_installed(self._install_dir_on_pythonpath, self._python)
output = release.get_cmd_stdout(
self._install_dir_on_pythonpath,
self._easy_install_cmd + ["-d", self._install_dir,
self._project_name])
# easy_install doesn't fail properly :-(
if "SyntaxError" in output:
raise Exception(output)
check_version_equals(self._install_dir_on_pythonpath,
self._expected_version,
self._python)
def test(self, log):
self._install_dir_on_pythonpath.cmd(self._test_cmd)
@action_tree.action_node
def easy_install_test(self):
return [
self.easy_install,
self.test,
]
def make_source_dist_easy_install_test_step(env, install_dir,
source_dir,
test_cmd, expected_version,
python_version):
python = "python%d.%d" % python_version
tester = EasyInstallTester(
env,
install_dir,
project_name=".",
test_cmd=test_cmd,
expected_version=expected_version,
easy_install_cmd=(cmd_env.in_dir(source_dir) +
[python, "setup.py", "easy_install"]),
python=python)
return tester.easy_install_test
def make_pypi_easy_install_test_step(env, install_dir,
test_cmd, expected_version,
python_version):
easy_install = "easy_install-%d.%d" % python_version
python = "python%d.%d" % python_version
tester = EasyInstallTester(
env,
install_dir,
project_name="mechanize",
test_cmd=test_cmd,
expected_version=expected_version,
easy_install_cmd=[easy_install],
python=python)
return tester.easy_install_test
def make_tarball_easy_install_test_step(env, install_dir,
tarball_path,
test_cmd, expected_version,
python_version):
easy_install = "easy_install-%d.%d" % python_version
python = "python%d.%d" % python_version
tester = EasyInstallTester(
env,
install_dir,
project_name=tarball_path,
test_cmd=test_cmd,
expected_version=expected_version,
easy_install_cmd=[easy_install],
python=python)
return tester.easy_install_test
class Releaser(object):
def __init__(self, env, git_repository_path, release_area, mirror_path,
build_tools_repo_path=None, run_in_repository=False,
tag_name=None, test_uri=None):
self._release_area = release_area
self._release_dir = release_dir = os.path.join(release_area, "release")
self._opt_dir = os.path.join(release_dir, "opt")
self._bin_dir = os.path.join(self._opt_dir, "bin")
AddToPathEnv = release.make_env_maker(release.add_to_path_cmd)
self._env = AddToPathEnv(release.GitPagerWrapper(env), self._bin_dir)
self._source_repo_path = git_repository_path
self._in_source_repo = release.CwdEnv(self._env,
self._source_repo_path)
self._tag_name = tag_name
self._set_next_release_version()
self._clone_path = os.path.join(release_dir, "clone")
self._in_clone = release.CwdEnv(self._env, self._clone_path)
if run_in_repository:
self._in_repo = self._in_source_repo
self._repo_path = self._source_repo_path
else:
self._in_repo = self._in_clone
self._repo_path = self._clone_path
self._docs_dir = os.path.join(self._repo_path, "docs")
self._in_docs_dir = release.CwdEnv(self._env, self._docs_dir)
self._in_release_dir = release.CwdEnv(self._env, self._release_dir)
self._build_tools_path = build_tools_repo_path
if self._build_tools_path is not None:
self._website_source_path = os.path.join(self._build_tools_path,
"website")
self._mirror_path = mirror_path
self._in_mirror = release.CwdEnv(self._env, self._mirror_path)
self._css_validator_path = "css_validator"
self._test_uri = test_uri
self._test_deps_dir = os.path.join(release_dir, "test_deps")
self._easy_install_test_dir = os.path.join(release_dir,
"easy_install_test")
self._in_easy_install_dir = release.CwdEnv(self._env,
self._easy_install_test_dir)
# prevent anything other than functional test dependencies being on
# sys.path due to cwd or PYTHONPATH
self._easy_install_env = clean_environ_env(
release.CwdEnv(env, self._test_deps_dir))
self._zope_testbrowser_dir = os.path.join(release_dir,
"zope_testbrowser_test")
def _mkdtemp(self):
temp_dir = tempfile.mkdtemp(prefix="tmp-%s-" % self.__class__.__name__)
def tear_down():
shutil.rmtree(temp_dir)
return temp_dir, tear_down
def _get_next_release_version(self):
# --pretend / git not installed
most_recent, next = "dummy version", "dummy version"
try:
tags = release.get_cmd_stdout(self._in_source_repo,
["git", "tag", "-l"]).split()
except cmd_env.CommandFailedError:
pass
else:
versions = [release.parse_version(tag) for tag in tags]
if versions:
most_recent = max(versions)
next = most_recent.next_version()
return most_recent, next
def _set_next_release_version(self):
self._previous_version, self._release_version = \
self._get_next_release_version()
if self._tag_name is not None:
self._release_version = release.parse_version(self._tag_name)
self._source_distributions = self._get_source_distributions(
self._release_version)
def _get_source_distributions(self, version):
def dist_basename(version, format):
return "mechanize-%s.%s" % (version, format)
return set([dist_basename(version, "zip"),
dist_basename(version, "tar.gz")])
def git_fetch(self, log):
# for tags
self._in_source_repo.cmd(["git", "fetch"])
self._set_next_release_version()
def print_next_tag(self, log):
print self._release_version
def _verify_version(self, path):
if str(self._release_version) not in \
release.read_file_from_env(self._in_repo, path):
raise MissingVersionError(path, self._release_version)
def _verify_versions(self):
for path in ["ChangeLog", "mechanize/_version.py"]:
self._verify_version(path)
def clone(self, log):
self._env.cmd(["git", "clone",
self._source_repo_path, self._clone_path])
def checks(self, log):
self._verify_versions()
def _ensure_installed(self, package_name, ppa):
release.ensure_installed(self._env,
cmd_env.PrefixCmdEnv(["sudo"], self._env),
package_name,
ppa=ppa)
def install_css_validator_in_release_area(self, log):
jar_dir = os.path.join(self._release_area, self._css_validator_path)
release.clean_dir(self._env, jar_dir)
in_jar_dir = release.CwdEnv(self._env, jar_dir)
in_jar_dir.cmd([
"wget",
"http://www.w3.org/QA/Tools/css-validator/css-validator.jar"])
in_jar_dir.cmd(["wget",
"http://jigsaw.w3.org/Distrib/jigsaw_2.2.6.tar.bz2"])
in_jar_dir.cmd(["sh", "-c", "tar xf jigsaw_*.tar.bz2"])
in_jar_dir.cmd(["ln", "-s", "Jigsaw/classes/jigsaw.jar"])
@action_tree.action_node
def install_deps(self):
dependency_actions = []
standard_dependency_actions = []
def add_dependency(package_name, ppa=None):
if ppa is None:
actions = standard_dependency_actions
else:
actions = dependency_actions
actions.append(
(package_name.replace(".", ""),
lambda log: self._ensure_installed(package_name, ppa)))
add_dependency("python2.6")
# required, but ubuntu doesn't have them any more :-( I installed these
# (and zope.interface and twisted SVN trunk) by hand
# add_dependency("python2.4"),
# add_dependency("python2.5")
# add_dependency("python2.7")
add_dependency("python-setuptools")
add_dependency("git-core")
# for running zope_testbrowser tests
add_dependency("python-virtualenv")
add_dependency("python2.6-dev")
# for deployment to SF and local collation of files for release
add_dependency("rsync")
# for running functional tests against local web server
add_dependency("python-twisted-web2")
# for generating .html docs from .txt markdown files
add_dependency("pandoc")
# for generating docs from .in templates
add_dependency("python-empy")
# for post-processing generated HTML
add_dependency("python-lxml")
# for the validate command
add_dependency("wdg-html-validator")
# for collecting code coverage data and generating coverage reports
# no 64 bit .deb ATM
# add_dependency("python-figleaf", ppa="jjl/figleaf")
# for css validator
add_dependency("default-jre")
add_dependency("libcommons-collections3-java")
add_dependency("libcommons-lang-java")
add_dependency("libxerces2-java")
add_dependency("libtagsoup-java")
# OMG, it depends on piles of java web server stuff, even for local
# command-line validation. You're doing it wrong!
add_dependency("velocity")
dependency_actions.append(self.install_css_validator_in_release_area)
dependency_actions.insert(0, action_tree.make_node(
standard_dependency_actions, "standard_dependencies"))
return dependency_actions
def copy_test_dependencies(self, log):
# so test.py can be run without the mechanize alongside it being on
# sys.path
# TODO: move mechanize package into a top-level directory, so it's not
# automatically on sys.path
def copy_in(src):
self._env.cmd(["cp", "-r", src, self._test_deps_dir])
release.clean_dir(self._env, self._test_deps_dir)
copy_in(os.path.join(self._repo_path, "test.py"))
copy_in(os.path.join(self._repo_path, "test"))
copy_in(os.path.join(self._repo_path, "test-tools"))
copy_in(os.path.join(self._repo_path, "examples"))
def _make_test_cmd(self, python_version,
local_server=True,
uri=None,
coverage=False):
python = "python%d.%d" % python_version
if coverage:
# python-figleaf only supports Python 2.6 ATM
assert python_version == (2, 6), python_version
python = "figleaf"
test_cmd = [python, "test.py"]
if not local_server:
test_cmd.append("--no-local-server")
# running against wwwsearch.sourceforge.net is slow, want to
# see where it failed
test_cmd.append("-v")
if coverage:
# TODO: Fix figleaf traceback with doctests
test_cmd.append("--skip-doctests")
if uri is not None:
test_cmd.extend(["--uri", uri])
return test_cmd
def performance_test(self, log):
result = run_performance_tests(self._repo_path)
if not result.wasSuccessful():
raise Exception("performance tests failed")
def clean_coverage(self, log):
self._in_repo.cmd(["rm", "-f", ".figleaf"])
self._in_repo.cmd(release.rm_rf_cmd("html"))
def _make_test_step(self, env, **kwds):
test_cmd = self._make_test_cmd(**kwds)
def test_step(log):
env.cmd(test_cmd)
return test_step
def _make_easy_install_test_cmd(self, **kwds):
test_cmd = self._make_test_cmd(**kwds)
test_cmd.extend(["discover", "--start-directory", self._test_deps_dir])
return test_cmd
def _make_source_dist_easy_install_test_step(self, env, **kwds):
test_cmd = self._make_easy_install_test_cmd(**kwds)
return make_source_dist_easy_install_test_step(
self._easy_install_env, self._easy_install_test_dir,
self._repo_path, test_cmd, self._release_version,
kwds["python_version"])
def _make_pypi_easy_install_test_step(self, env, **kwds):
test_cmd = self._make_easy_install_test_cmd(**kwds)
return make_pypi_easy_install_test_step(
self._easy_install_env, self._easy_install_test_dir,
test_cmd, self._release_version, kwds["python_version"])
def _make_tarball_easy_install_test_step(self, env, **kwds):
test_cmd = self._make_easy_install_test_cmd(**kwds)
[tarball] = list(d for d in self._source_distributions if
d.endswith(".tar.gz"))
return make_tarball_easy_install_test_step(
self._easy_install_env, self._easy_install_test_dir,
os.path.abspath(os.path.join(self._repo_path, "dist", tarball)),
test_cmd, self._release_version, kwds["python_version"])
def _make_unpacked_tarball_test_step(self, env, **kwds):
# This catches mistakes in listing test files in MANIFEST.in (the tests
# don't get installed, so these don't get caught by testing installed
# code).
test_cmd = self._make_test_cmd(**kwds)
[tarball] = list(d for d in self._source_distributions if
d.endswith(".tar.gz"))
tarball_path = os.path.abspath(
os.path.join(self._repo_path, "dist", tarball))
def test_step(log):
target_dir, tear_down = self._mkdtemp()
try:
env.cmd(["tar", "-C", target_dir, "-xf", tarball_path])
[source_dir] = glob.glob(
os.path.join(target_dir, "mechanize-*"))
test_env = clean_environ_env(release.CwdEnv(env, source_dir))
test_env.cmd(test_cmd)
finally:
tear_down()
return test_step
@action_tree.action_node
def test(self):
r = []
r.append(("python27_test",
self._make_test_step(self._in_repo, python_version=(2, 7))))
r.append(("python27_easy_install_test",
self._make_source_dist_easy_install_test_step(
self._in_repo, python_version=(2, 7))))
r.append(("python26_test",
self._make_test_step(self._in_repo, python_version=(2, 6))))
# disabled for the moment -- think I probably built the launchpad .deb
# from wrong branch, without bug fixes
# r.append(("python26_coverage",
# self._make_test_step(self._in_repo, python_version=(2, 6),
# coverage=True)))
r.append(("python25_easy_install_test",
self._make_source_dist_easy_install_test_step(
self._in_repo, python_version=(2, 5))))
r.append(("python24_easy_install_test",
self._make_source_dist_easy_install_test_step(
self._in_repo, python_version=(2, 4))))
r.append(self.performance_test)
return r
def make_coverage_html(self, log):
self._in_repo.cmd(["figleaf2html"])
def tag(self, log):
self._in_repo.cmd(["git", "checkout", "master"])
self._in_repo.cmd(["git", "tag",
"-m", "Tagging release %s" % self._release_version,
str(self._release_version)])
def clean_docs(self, log):
self._in_docs_dir.cmd(release.rm_rf_cmd("html"))
def make_docs(self, log):
self._in_docs_dir.cmd(["mkdir", "-p", "html"])
site_map = release.site_map()
def pandoc(filename, source_filename):
last_modified = release.last_modified(source_filename,
self._in_docs_dir)
if filename == "download.txt":
last_modified = time.gmtime()
variables = [
("last_modified_iso",
time.strftime("%Y-%m-%d", last_modified)),
("last_modified_month_year",
time.strftime("%B %Y", last_modified))]
page_name = os.path.splitext(os.path.basename(filename))[0]
variables.append(("nav", release.nav_html(site_map, page_name)))
variables.append(("subnav", release.subnav_html(site_map,
page_name)))
release.pandoc(self._in_docs_dir, filename, variables=variables)
release.empy(self._in_docs_dir, "forms.txt.in")
release.empy(self._in_docs_dir, "download.txt.in",
defines=["version=%r" % str(self._release_version)])
for page in site_map.iter_pages():
if page.name in ["Root", "Changelog"]:
continue
source_filename = filename = page.name + ".txt"
if page.name in ["forms", "download"]:
source_filename += ".in"
pandoc(filename, source_filename)
self._in_repo.cmd(["cp", "-r", "ChangeLog", "docs/html/ChangeLog.txt"])
if self._build_tools_path is not None:
styles = release.ensure_trailing_slash(
os.path.join(self._website_source_path, "styles"))
self._env.cmd(["rsync", "-a", styles,
os.path.join(self._docs_dir, "styles")])
def setup_py_sdist(self, log):
self._in_repo.cmd(release.rm_rf_cmd("dist"))
# write empty setup.cfg so source distribution is built using a version
# number without ".dev" and today's date appended
self._in_repo.cmd(cmd_env.write_file_cmd("setup.cfg", ""))
self._in_repo.cmd(["python", "setup.py", "sdist",
"--formats=gztar,zip"])
archives = set(os.listdir(os.path.join(self._repo_path, "dist")))
assert archives == self._source_distributions, \
(archives, self._source_distributions)
@action_tree.action_node
def build_sdist(self):
return [
self.clean_docs,
self.make_docs,
self.setup_py_sdist,
]
def _stage(self, path, dest_dir, dest_basename=None,
source_base_path=None):
# IIRC not using rsync because didn't see easy way to avoid updating
# timestamp of unchanged files, which was upsetting git
# note: files in the website repository that are no longer generated
# must be manually deleted from the repository
if source_base_path is None:
source_base_path = self._repo_path
full_path = os.path.join(source_base_path, path)
try:
self._env.cmd(["readlink", "-e", full_path],
stdout=open(os.devnull, "w"))
except cmd_env.CommandFailedError:
print "not staging (does not exist):", full_path
return
if dest_basename is None:
dest_basename = os.path.basename(path)
dest = os.path.join(self._mirror_path, dest_dir, dest_basename)
try:
self._env.cmd(["cmp", full_path, dest])
except cmd_env.CommandFailedError:
print "staging: %s -> %s" % (full_path, dest)
self._env.cmd(["cp", full_path, dest])
else:
print "not staging (unchanged): %s -> %s" % (full_path, dest)
def ensure_unmodified(self, log):
if self._build_tools_path:
release.ensure_unmodified(self._env, self._website_source_path)
release.ensure_unmodified(self._env, self._mirror_path)
def _stage_flat_dir(self, path, dest):
self._env.cmd(["mkdir", "-p", os.path.join(self._mirror_path, dest)])
for filename in os.listdir(path):
self._stage(os.path.join(path, filename), dest)
def _symlink_flat_dir(self, path, exclude):
for filename in os.listdir(path):
if filename in exclude:
continue
link_dir = os.path.dirname(path)
target = os.path.relpath(os.path.join(path, filename), link_dir)
link_path = os.path.join(link_dir, filename)
if not os.path.islink(link_path) or \
os.path.realpath(link_path) != target:
self._env.cmd(["ln", "-f", "-s", "-t", link_dir, target])
def collate_from_mechanize(self, log):
html_dir = os.path.join(self._docs_dir, "html")
self._stage_flat_dir(html_dir, "htdocs/mechanize/docs")
self._symlink_flat_dir(
os.path.join(self._mirror_path, "htdocs/mechanize/docs"),
exclude=[".git", ".htaccess", ".svn", "CVS"])
self._stage("test-tools/cookietest.cgi", "cgi-bin")
self._stage("examples/forms/echo.cgi", "cgi-bin")
self._stage("examples/forms/example.html", "htdocs/mechanize")
for archive in self._source_distributions:
placeholder = os.path.join("htdocs/mechanize/src", archive)
self._in_mirror.cmd(["touch", placeholder])
def collate_from_build_tools(self, log):
self._stage(os.path.join(self._website_source_path, "frontpage.html"),
"htdocs", "index.html")
self._stage_flat_dir(
os.path.join(self._website_source_path, "styles"), "htdocs/styles")
@action_tree.action_node
def collate(self):
r = [self.collate_from_mechanize]
if self._build_tools_path is not None:
r.append(self.collate_from_build_tools)
return r
def collate_pypi_upload_built_items(self, log):
for archive in self._source_distributions:
self._stage(os.path.join("dist", archive), "htdocs/mechanize/src")
def commit_staging_website(self, log):
self._in_mirror.cmd(["git", "add", "--all"])
self._in_mirror.cmd(
["git", "commit",
"-m", "Automated update for release %s" % self._release_version])
def validate_html(self, log):
exclusions = set(f for f in """\
./cookietest.html
htdocs/basic_auth/index.html
htdocs/digest_auth/index.html
htdocs/mechanize/example.html
htdocs/test_fixtures/index.html
htdocs/test_fixtures/mechanize_reload_test.html
htdocs/test_fixtures/referertest.html
""".splitlines() if not f.startswith("#"))
for dirpath, dirnames, filenames in os.walk(self._mirror_path):
try:
# archived website
dirnames.remove("old")
except ValueError:
pass
for filename in filenames:
if filename.endswith(".html"):
page_path = os.path.join(
os.path.relpath(dirpath, self._mirror_path), filename)
if page_path not in exclusions:
self._in_mirror.cmd(["validate", page_path])
def _classpath_cmd(self):
from_packages = ["/usr/share/java/commons-collections3.jar",
"/usr/share/java/commons-lang.jar",
"/usr/share/java/xercesImpl.jar",
"/usr/share/java/tagsoup.jar",
"/usr/share/java/velocity.jar",
]
jar_dir = os.path.join(self._release_area, self._css_validator_path)
local = glob.glob(os.path.join(jar_dir, "*.jar"))
path = ":".join(local + from_packages)
return ["env", "CLASSPATH=%s" % path]
def _sanitise_css(self, path):
temp_dir, tear_down = self._mkdtemp()
temp_path = os.path.join(temp_dir, os.path.basename(path))
temp = open(temp_path, "w")
try:
for line in open(path):
if line.rstrip().endswith("/*novalidate*/"):
# temp.write("/*%s*/\n" % line.rstrip())
temp.write("/*sanitised*/\n")
else:
temp.write(line)
finally:
temp.close()
return temp_path, tear_down
def validate_css(self, log):
env = cmd_env.PrefixCmdEnv(self._classpath_cmd(), self._in_release_dir)
# env.cmd(["java", "org.w3c.css.css.CssValidator", "--help"])
"""
Usage: java org.w3c.css.css.CssValidator [OPTIONS] | [URL]*
OPTIONS
-p, --printCSS
Prints the validated CSS (only with text output, the CSS is printed with other outputs)
-profile PROFILE, --profile=PROFILE
Checks the Stylesheet against PROFILE
Possible values for PROFILE are css1, css2, css21 (default), css3, svg, svgbasic, svgtiny, atsc-tv, mobile, tv
-medium MEDIUM, --medium=MEDIUM
Checks the Stylesheet using the medium MEDIUM
Possible values for MEDIUM are all (default), aural, braille, embossed, handheld, print, projection, screen, tty, tv, presentation
-output OUTPUT, --output=OUTPUT
Prints the result in the selected format
Possible values for OUTPUT are text (default), xhtml, html (same result as xhtml), soap12
-lang LANG, --lang=LANG
Prints the result in the specified language
Possible values for LANG are de, en (default), es, fr, ja, ko, nl, zh-cn, pl, it
-warning WARN, --warning=WARN
Warnings verbosity level
Possible values for WARN are -1 (no warning), 0, 1, 2 (default, all the warnings
URL
URL can either represent a distant web resource (http://) or a local file (file:/)
"""
validate_cmd = ["java", "org.w3c.css.css.CssValidator"]
for dirpath, dirnames, filenames in os.walk(self._mirror_path):
for filename in filenames:
if filename.endswith(".css"):
path = os.path.join(dirpath, filename)
temp_path, tear_down = self._sanitise_css(path)
try:
page_url = "file://" + temp_path
output = release.get_cmd_stdout(
env, validate_cmd + [page_url])
finally:
tear_down()
# the validator doesn't fail properly: it exits
# successfully on validation failure
if "Sorry! We found the following errors" in output:
raise CSSValidationError(path, output)
def fetch_zope_testbrowser(self, log):
release.clean_dir(self._env, self._zope_testbrowser_dir)
in_testbrowser = release.CwdEnv(self._env, self._zope_testbrowser_dir)
in_testbrowser.cmd(["easy_install", "--editable",
"--build-directory", ".",
"zope.testbrowser[test]"])
in_testbrowser.cmd(
["virtualenv", "--no-site-packages", "zope.testbrowser"])
project_dir = os.path.join(self._zope_testbrowser_dir,
"zope.testbrowser")
in_project_dir = clean_environ_env(
release.CwdEnv(self._env, project_dir))
check_not_installed(in_project_dir, "bin/python")
in_project_dir.cmd(
["sed", "-i", "-e", "s/mechanize[^\"']*/mechanize/", "setup.py"])
in_project_dir.cmd(["bin/easy_install", "zc.buildout"])
in_project_dir.cmd(["bin/buildout", "init"])
[mechanize_tarball] = list(d for d in self._source_distributions if
d.endswith(".tar.gz"))
tarball_path = os.path.join(self._repo_path, "dist", mechanize_tarball)
in_project_dir.cmd(["bin/easy_install", tarball_path])
in_project_dir.cmd(["bin/buildout", "install"])
def test_zope_testbrowser(self, log):
project_dir = os.path.join(self._zope_testbrowser_dir,
"zope.testbrowser")
env = clean_environ_env(release.CwdEnv(self._env, project_dir))
check_version_equals(env, self._release_version, "bin/python")
env.cmd(["bin/test"])
@action_tree.action_node
def zope_testbrowser(self):
return [self.fetch_zope_testbrowser,
self.test_zope_testbrowser,
]
def upload_to_pypi(self, log):
self._in_repo.cmd(["python", "setup.py", "sdist",
"--formats=gztar,zip", "upload"])
def sync_to_sf(self, log):
assert os.path.isdir(
os.path.join(self._mirror_path, "htdocs/mechanize"))
self._env.cmd(["rsync", "-rlptvuz", "--exclude", "*~", "--delete",
release.ensure_trailing_slash(self._mirror_path),
"jjlee,wwwsearch@web.sourceforge.net:"])
@action_tree.action_node
def upload(self):
r = []
r.append(self.upload_to_pypi)
# setup.py upload requires sdist command to upload zip files, and the
# sdist comment insists on rebuilding source distributions, so it's not
# possible to use the upload command to upload the already-built zip
# file. Work around that by copying the rebuilt source distributions
# into website repository only now (rather than at build/test time), so
# don't end up with two different sets of source distributions with
# different md5 sums due to timestamps in the archives.
r.append(self.collate_pypi_upload_built_items)
r.append(self.commit_staging_website)
if self._mirror_path is not None:
r.append(self.sync_to_sf)
return r
def clean(self, log):
release.clean_dir(self._env, self._release_area)
def clean_most(self, log):
# not dependencies installed in release area (css validator)
release.clean_dir(self._env, self._release_dir)
def write_email(self, log):
log = release.get_cmd_stdout(self._in_repo,
["git", "log", '--pretty=format: * %s',
"%s..HEAD" % self._previous_version])
# filter out some uninteresting commits
log = "".join(line for line in log.splitlines(True) if not
re.match("^ \* Update (?:changelog|version)$", line,
re.I))
self._in_release_dir.cmd(cmd_env.write_file_cmd(
"announce_email.txt", u"""\
ANN: mechanize {version} released
http://wwwsearch.sourceforge.net/mechanize/
This is a stable bugfix release.
Changes since {previous_version}:
{log}
About mechanize
=============================================
Requires Python 2.4, 2.5, 2.6, or 2.7.
Stateful programmatic web browsing, after Andy Lester's Perl module
WWW::Mechanize.
Example:
import re
from mechanize import Browser
b = Browser()
b.open("http://www.example.com/")
# follow second link with element text matching regular expression
response = b.follow_link(text_regex=re.compile(r"cheese\s*shop"), nr=1)
b.select_form(name="order")
# Browser passes through unknown attributes (including methods)
# to the selected HTMLForm
b["cheeses"] = ["mozzarella", "caerphilly"] # (the method here is __setitem__)
response2 = b.submit() # submit current form
response3 = b.back() # back to cheese shop
response4 = b.reload()
for link in b.forms():
print form
# .links() optionally accepts the keyword args of .follow_/.find_link()
for link in b.links(url_regex=re.compile("python.org")):
print link
b.follow_link(link) # can be EITHER Link instance OR keyword args
b.back()
John
""".format(log=log,
version=self._release_version,
previous_version=self._previous_version)))
def edit_email(self, log):
self._in_release_dir.cmd(["sensible-editor", "announce_email.txt"])
def push_tag(self, log):
self._in_repo.cmd(["git", "push", "git@github.com:jjlee/mechanize.git",
"tag", str(self._release_version)])
def send_email(self, log):
text = release.read_file_from_env(self._in_release_dir,
"announce_email.txt")
subject, sep, body = text.partition("\n")
body = body.lstrip()
assert len(body) > 0, body
release.send_email(
from_address="John J Lee <jjl@pobox.com>",
to_address="wwwsearch-general@lists.sourceforge.net",
subject=subject,
body=body)
@action_tree.action_node
def build(self):
return [
self.clean,
self.install_deps,
self.clean_most,
self.git_fetch,
self.print_next_tag,
self.clone,
self.checks,
# self.clean_coverage,
self.copy_test_dependencies,
self.test,
# self.make_coverage_html,
self.tag,
self.build_sdist,
("unpacked_tarball_test", self._make_unpacked_tarball_test_step(
self._env, python_version=(2,6))),
("easy_install_test", self._make_tarball_easy_install_test_step(
self._in_repo, python_version=(2, 6),
local_server=False, uri=self._test_uri)),
self.zope_testbrowser,
self.write_email,
self.edit_email,
]
def update_version(self, log):
version_path = "mechanize/_version.py"
template = """\
"%(text)s"
__version__ = %(tuple)s
"""
old_text = release.read_file_from_env(self._in_source_repo,
version_path)
old_version = old_text.splitlines()[0].strip(' "')
assert old_version == str(self._release_version), \
(old_version, str(self._release_version))
def version_text(version):
return template % {"text": str(version),
"tuple": repr(tuple(version.tuple[:-1]))}
assert old_text == version_text(release.parse_version(old_version)), \
(old_text, version_text(release.parse_version(old_version)))
self._in_source_repo.cmd(cmd_env.write_file_cmd(
version_path,
version_text(self._release_version.next_version())))
self._in_source_repo.cmd(["git", "commit", "-m", "Update version",
version_path])
@action_tree.action_node
def update_staging_website(self):
if self._mirror_path is None:
return []
return [
self.ensure_unmodified,
self.collate,
self.validate_html,
self.validate_css,
self.commit_staging_website,
]
@action_tree.action_node
def tell_the_world(self):