From 5d04ed125607943ac8bb26bc97fb173c2fa54407 Mon Sep 17 00:00:00 2001 From: Toby Date: Mon, 12 Mar 2012 12:59:52 +0000 Subject: [PATCH 1/6] add new profiling_testing --- ckan/ckan_nose_plugin.py | 24 ++++++++++++++++++++++++ ckan/config/middleware.py | 1 + ckan/controllers/user.py | 4 ++-- ckan/i18n/bg/LC_MESSAGES/ckan.mo | Bin 68523 -> 68524 bytes ckan/i18n/ca/LC_MESSAGES/ckan.mo | Bin 52848 -> 52848 bytes ckan/i18n/cs_CZ/LC_MESSAGES/ckan.mo | Bin 50557 -> 50557 bytes ckan/i18n/de/LC_MESSAGES/ckan.mo | Bin 51913 -> 51913 bytes ckan/i18n/el/LC_MESSAGES/ckan.mo | Bin 53912 -> 53912 bytes ckan/i18n/es/LC_MESSAGES/ckan.mo | Bin 53313 -> 53313 bytes ckan/i18n/fi/LC_MESSAGES/ckan.mo | Bin 52031 -> 52031 bytes ckan/i18n/fr/LC_MESSAGES/ckan.mo | Bin 55030 -> 55030 bytes ckan/i18n/hu/LC_MESSAGES/ckan.mo | Bin 50478 -> 50478 bytes ckan/i18n/it/LC_MESSAGES/ckan.mo | Bin 51887 -> 51887 bytes ckan/i18n/lt/LC_MESSAGES/ckan.mo | Bin 48768 -> 48768 bytes ckan/i18n/nl/LC_MESSAGES/ckan.mo | Bin 49368 -> 49368 bytes ckan/i18n/no/LC_MESSAGES/ckan.mo | Bin 50123 -> 50123 bytes ckan/i18n/pl/LC_MESSAGES/ckan.mo | Bin 52262 -> 52262 bytes ckan/i18n/pt_BR/LC_MESSAGES/ckan.mo | Bin 53134 -> 53134 bytes ckan/i18n/ro/LC_MESSAGES/ckan.mo | Bin 48744 -> 48744 bytes ckan/i18n/ru/LC_MESSAGES/ckan.mo | Bin 63191 -> 63191 bytes ckan/i18n/sl/LC_MESSAGES/ckan.mo | Bin 49023 -> 49023 bytes ckan/i18n/sq/LC_MESSAGES/ckan.mo | Bin 49985 -> 49985 bytes ckan/i18n/sr/LC_MESSAGES/ckan.mo | Bin 61796 -> 61796 bytes ckan/i18n/sr_Latn/LC_MESSAGES/ckan.mo | Bin 51563 -> 51563 bytes ckan/i18n/sv/LC_MESSAGES/ckan.mo | Bin 50305 -> 50305 bytes ckan/lib/helpers.py | 1 + ckan/templates/home/about.html | 1 + ckan/templates/layout_base.html | 2 +- profile_tests.py | 4 ++++ 29 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 profile_tests.py diff --git a/ckan/ckan_nose_plugin.py b/ckan/ckan_nose_plugin.py index 0666b7761c3..58b9d0061a0 100644 --- a/ckan/ckan_nose_plugin.py +++ b/ckan/ckan_nose_plugin.py @@ -5,6 +5,7 @@ import pkg_resources from paste.deploy import loadapp from pylons import config +import time class CkanNose(Plugin): settings = None @@ -65,3 +66,26 @@ def describeTest(self, test): # display module name instead of docstring return False + def startTest(self, test): + """ + startTest: start timing. + """ + self._started = time.time() + + def stopTest(self, test): + """ + stopTest: stop timing, canonicalize the test name, and save + the run time. + """ + runtime = time.time() - self._started + + # CTB: HACK! + f = open('times.txt', 'a') + + testname = str(test) + #if ' ' in testname: + # testname = testname.split(' ')[1] + + f.write('%s,%s\n' % (testname, str(runtime))) + + f.close() diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py index 15c092877e2..48b68b16842 100644 --- a/ckan/config/middleware.py +++ b/ckan/config/middleware.py @@ -109,6 +109,7 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf): # Serve static files static_max_age = None if not asbool(config.get('ckan.cache_enabled')) \ else int(config.get('ckan.static_max_age', 3600)) + print static_max_age static_app = StaticURLParser(config['pylons.paths']['static_files'], cache_max_age=static_max_age) diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index f446c21c790..88dee13b4fa 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -130,8 +130,8 @@ def new(self, data=None, errors=None, error_summary=None): if c.user and not data: # #1799 Don't offer the registration form if already logged in - return render('user/logout_first.html') - + return render('user/logout_first.html') + data = data or {} errors = errors or {} error_summary = error_summary or {} diff --git a/ckan/i18n/bg/LC_MESSAGES/ckan.mo b/ckan/i18n/bg/LC_MESSAGES/ckan.mo index e8603d315ffa2d513c83ac958935bd7d829b3932..8a8a2acd1ce26c0d5d248197a05e9d398e382e30 100644 GIT binary patch delta 567 zcmXZZJuE{}6u|Lw>s5I~Qy=kJ{Ro=+NK^;3gv3Z9)j?8`7_fA}U?QGPEsRaoJ3Mc!MV1VJ|)+^NHb55<~}%V+i&B zHHW37onVJTGag|JUScigu@N7z1K-dH$Qv4pga@;Qb!%TmvdNE0*Dr36OBnzF delta 565 zcmXZZPbkA-7{Kx8-F`MkD=hP8W){;HWecqhEk{XADWnv+kT|%Aem8#(T;)!wIZQig z<-kF$9L2$fgNt1iHh;daskh(fd7j?i+w;8d*`bj-G}0cwIb;zz{SbLmBCiFJMiF`X z5}Cjvmf_g9$P!NCI9_1@T|Xj8jAJJrAWO1*uC=0M6HTT5{wrdXeAJ?38h0$plo^6& z3UM5?DoNl5&S4gJY0P6NStCCwQ?ib3o04YS#~Mtb6ED$$w-~|)WInOll?2g-1K5pv z|BBsI(nPRD!Hq{)kLOs0IdtJYdN7X$R+N|aJF%ZUfQvYfx`7&5D1kmWhWc%%GQqZs5l5se(&Gz9V!h!^e%0K|EPp1de*ux*N}>P& diff --git a/ckan/i18n/ca/LC_MESSAGES/ckan.mo b/ckan/i18n/ca/LC_MESSAGES/ckan.mo index 0fbf413cb40c7b2c5b977252886fb7d93e3fac88..b231709129f61cf02b51de7d40b7d1e7f6a31229 100644 GIT binary patch delta 25 hcmew`hxx-C<_(VuxQulTjT8)xtc(pdzbR;m1ptbt3H|^8 delta 25 hcmew`hxx-C<_(VuxQuj-OcV@_tV|6zzbR;m1ptcC3IhND diff --git a/ckan/i18n/cs_CZ/LC_MESSAGES/ckan.mo b/ckan/i18n/cs_CZ/LC_MESSAGES/ckan.mo index d2e0a9cb04d2692fbeffd0750cdbd9c84f78821a..c3962d6b0f82cbc27b54cd3d674e4fab1fdee725 100644 GIT binary patch delta 25 hcmey{#r(I6dBdXuE@NFoBLzbvD`SJrZwe-c004>c3HSg2 delta 25 hcmey{#r(I6dBdXuE+btd69q#fD^tVGZwe-c004>`3H<;7 diff --git a/ckan/i18n/de/LC_MESSAGES/ckan.mo b/ckan/i18n/de/LC_MESSAGES/ckan.mo index 18b956def3e03d62e7051fa409910629ae06dddb..3b792ed1c6882d7213c3962b8d50acef3e29a8cc 100644 GIT binary patch delta 25 hcmX>(mHFgU<_(VuxQulTjT8)xtc(pdzbUvI2>^oe3HJa1 delta 25 hcmX>(mHFgU<_(VuxQuj-OcV@_tV|6zzbUvI2>^o|3H$&6 diff --git a/ckan/i18n/el/LC_MESSAGES/ckan.mo b/ckan/i18n/el/LC_MESSAGES/ckan.mo index 60e2f5f43e23aa54aacfb27db15fccd941042ef6..6c33b3abe608ac01d62a41b0e8966f75cc2f907f 100644 GIT binary patch delta 25 hcmbQSlzGNd<_(VuxQulTjT8)xtc(pdzbROg2mpL&35);$ delta 25 hcmbQSlzGNd<_(VuxQuj-OcV@_tV|6zzbROg2mpMN36TH* diff --git a/ckan/i18n/es/LC_MESSAGES/ckan.mo b/ckan/i18n/es/LC_MESSAGES/ckan.mo index 8e54d36ade3a8a80f52b1dd6cd14d1770ef64772..da0685018fefe56d85d3b7ea7cc895420053a81e 100644 GIT binary patch delta 25 hcmX@OfcfA8<_(VuxQulTjT8)xtc(pdzbOcf2LOUz33~tl delta 25 hcmX@OfcfA8<_(VuxQuj-OcV@_tV|6zzbOcf2LOVI34j0q diff --git a/ckan/i18n/fi/LC_MESSAGES/ckan.mo b/ckan/i18n/fi/LC_MESSAGES/ckan.mo index 5980d6f6a88796efbdce5f2f614840fb8ba32f6e..29c1493653305da5fabf2d1388dd3ce3e58bf8c6 100644 GIT binary patch delta 25 hcmdl#jd}kx<_(VuxQulTjT8)xtc(pdzbOcg0sw$D30wdG delta 25 hcmdl#jd}kx<_(VuxQuj-OcV@_tV|6zzbOcg0sw$t31I*L diff --git a/ckan/i18n/fr/LC_MESSAGES/ckan.mo b/ckan/i18n/fr/LC_MESSAGES/ckan.mo index 9581f1a762ef8ef8acb8296375b0f285df9cf3a6..ec80875c73a34357b61f8329b951e264a6a19223 100644 GIT binary patch delta 25 hcmeyimigOS<_(VuxQulTjT8)xtc(pdzbW8K0|1TF3A+FQ delta 25 hcmeyimigOS<_(VuxQuj-OcV@_tV|6zzbW8K0|1Tv3BUjV diff --git a/ckan/i18n/hu/LC_MESSAGES/ckan.mo b/ckan/i18n/hu/LC_MESSAGES/ckan.mo index 7911b27a98ae30287e0cb264f132afdc18a4283c..d0f83ac33bcc2d579ffa8759ce979017cb4c70dc 100644 GIT binary patch delta 25 gcmZ42#k{VIdBdXuE@NFoBLzbvD`SJrZwj130D9613jhEB delta 25 gcmZ42#k{VIdBdXuE+btd69q#fD^tVGZwj130DAlh5C8xG diff --git a/ckan/i18n/it/LC_MESSAGES/ckan.mo b/ckan/i18n/it/LC_MESSAGES/ckan.mo index 01aebc4bc4e11904b0364364fadf7866cbc4c8cb..73428446318d33b2d306caaebe9ca687b82c6388 100644 GIT binary patch delta 25 hcmZ2Km3jSC<_(VuxQulTjT8)xtc(pdzbQBz2>^Zf38?@8 delta 25 hcmZ2Km3jSC<_(VuxQuj-OcV@_tV|6zzbQBz2>^Z}39bMD diff --git a/ckan/i18n/lt/LC_MESSAGES/ckan.mo b/ckan/i18n/lt/LC_MESSAGES/ckan.mo index 57686dc12e298f0e969d0848ebc767c742665409..890980448afb8ee22d1e3d50a4bce03fe5adae8f 100644 GIT binary patch delta 25 hcmZqp%hd3fX~UxeE@NFoBLzbvD`SJrZwjXP0sw~z3Eltz delta 25 hcmZqp%hd3fX~UxeE+btd69q#fD^tVGZwjXP0sx0I3F80& diff --git a/ckan/i18n/nl/LC_MESSAGES/ckan.mo b/ckan/i18n/nl/LC_MESSAGES/ckan.mo index 8f9f9097bddb4a0632deb8053c90dd9afbde8c39..d43b055cc12f6e8f5ad7ea334bee074daf5bd63f 100644 GIT binary patch delta 25 hcmcc7$b6%bdBdXuE@NFoBLzbvD`SJrZwlV{0|0_T3Gx5{ delta 25 hcmcc7$b6%bdBdXuE+btd69q#fD^tVGZwlV{0|0_-3HJa1 diff --git a/ckan/i18n/no/LC_MESSAGES/ckan.mo b/ckan/i18n/no/LC_MESSAGES/ckan.mo index bebe704de324afc801ecfcb3c9e426e45a923f53..5eb5c0540b3441846ed9310c9327b577db6faec2 100644 GIT binary patch delta 25 hcmX@z&V0I^dBdXuE@NFoBLzbvD`SJrZwl@Q0RVv33EBVv delta 25 hcmX@z&V0I^dBdXuE+btd69q#fD^tVGZwl@Q0RVvj3Euz! diff --git a/ckan/i18n/pl/LC_MESSAGES/ckan.mo b/ckan/i18n/pl/LC_MESSAGES/ckan.mo index a6ff5efcad4ebaaacae62d55c5791f7ac4d2d35e..886c113107ff8382a0f430cc4ac77296979c12bc 100644 GIT binary patch delta 25 hcmZ2BgL&Bu<_(VuxQulTjT8)xtc(pdzbUYa1^|1<2^IhV delta 25 hcmZ2BgL&Bu<_(VuxQuj-OcV@_tV|6zzbUYa1^|2U2^#About ${g.site_title} +

What was the average price of a house in the UK in 1935? When will India's projected population overtake that of China? Where can you see publicly-funded art in Seattle? Data to answer many, many questions like these is out there on the Internet somewhere - but it is not always easy to find.

${g.site_title} is a community-run catalogue of useful sets of data on the Internet. You can collect links here to data from around the web for yourself and others to use, or search for data that others have collected. Depending on the type of data (and its conditions of use), ${g.site_title} may also be able to store a copy of the data or host it in a database, and provide some basic visualisation tools.

diff --git a/ckan/templates/layout_base.html b/ckan/templates/layout_base.html index 51fa8088c1c..bb7d8bed65a 100644 --- a/ckan/templates/layout_base.html +++ b/ckan/templates/layout_base.html @@ -27,7 +27,7 @@ - + diff --git a/profile_tests.py b/profile_tests.py new file mode 100644 index 00000000000..de653e536d0 --- /dev/null +++ b/profile_tests.py @@ -0,0 +1,4 @@ +import nose +import cProfile +command = """nose.main(argv=['--ckan','--with-pylons=test-core.ini', 'ckan/tests', '-x', '-v'])""" +cProfile.runctx( command, globals(), locals(), filename="ckan.profile" ) From 8cc806a0119b54da7f0eeb0220bf154c88721004 Mon Sep 17 00:00:00 2001 From: Toby Date: Thu, 15 Mar 2012 12:25:40 +0000 Subject: [PATCH 2/6] move _activity_div function into own include to increase render speed --- ckan/templates/_util.html | 29 ++-------------- ckan/templates/_util_activity_div.html | 34 +++++++++++++++++++ .../templates/activity_streams/added_tag.html | 2 +- .../activity_streams/changed_group.html | 2 +- .../activity_streams/changed_package.html | 3 +- .../changed_package_extra.html | 2 +- .../activity_streams/changed_resource.html | 2 +- .../activity_streams/changed_user.html | 2 +- .../activity_streams/deleted_group.html | 2 +- .../activity_streams/deleted_package.html | 2 +- .../deleted_package_extra.html | 2 +- .../activity_streams/deleted_resource.html | 2 +- .../templates/activity_streams/new_group.html | 2 +- .../activity_streams/new_package.html | 2 +- .../activity_streams/new_package_extra.html | 2 +- .../activity_streams/new_resource.html | 2 +- ckan/templates/activity_streams/new_user.html | 2 +- .../activity_streams/removed_tag.html | 2 +- 18 files changed, 53 insertions(+), 43 deletions(-) create mode 100644 ckan/templates/_util_activity_div.html diff --git a/ckan/templates/_util.html b/ckan/templates/_util.html index 791125aa9c9..08c2da49f65 100644 --- a/ckan/templates/_util.html +++ b/ckan/templates/_util.html @@ -6,6 +6,8 @@ py:strip="" > + + @@ -450,33 +452,6 @@ -
- - ${actor} - - - ${verb} - - - - ${object} - - - - - - - ${target} - - - - - - ${h.render_datetime(activity.timestamp)} - -
- + From dda53dfe069de983509972e7c61a7bedf4f41e66 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Fri, 16 Mar 2012 09:54:17 +0000 Subject: [PATCH 5/6] [#2226,doc][xs]: standardize naming of files and correct some formatting. --- ...{database_dumps.rst => database-dumps.rst} | 2 +- ...t => howto-editing-database-ownership.rst} | 0 doc/index.rst | 6 ++--- doc/{loading_data.rst => loading-data.rst} | 0 doc/paster.rst | 24 +++++++++---------- 5 files changed, 16 insertions(+), 16 deletions(-) rename doc/{database_dumps.rst => database-dumps.rst} (94%) rename doc/{editing_the_database_ownership.rst => howto-editing-database-ownership.rst} (100%) rename doc/{loading_data.rst => loading-data.rst} (100%) diff --git a/doc/database_dumps.rst b/doc/database-dumps.rst similarity index 94% rename from doc/database_dumps.rst rename to doc/database-dumps.rst index 5d5b6985a95..b8200d912d3 100644 --- a/doc/database_dumps.rst +++ b/doc/database-dumps.rst @@ -1,7 +1,7 @@ Database Dumps ============== -It's often useful to allow users to download a complete CKAN database in a dumpfile. For example, you can download thedatahub.org's daily dump at: http://thedatahub.org/dump/ in JSON format. The file is called ``ckan.net-daily.json.gz``. +It's often useful to allow users to download a complete CKAN database in a dumpfile. In addition, a CKAN administrator would like to easily backup and restore a CKAN database. diff --git a/doc/editing_the_database_ownership.rst b/doc/howto-editing-database-ownership.rst similarity index 100% rename from doc/editing_the_database_ownership.rst rename to doc/howto-editing-database-ownership.rst diff --git a/doc/index.rst b/doc/index.rst index 25da881ae86..1760bb68d46 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -39,7 +39,7 @@ Publishing Datasets .. toctree:: :maxdepth: 2 - loading_data + loading-data authorization storage/filestore storage/datastore @@ -60,8 +60,7 @@ General Administration :maxdepth: 2 paster - database_dumps - i18n + database-dumps configuration common-error-messages @@ -71,6 +70,7 @@ For CKAN Developers .. toctree:: :maxdepth: 2 + i18n test dev/buildbot diff --git a/doc/loading_data.rst b/doc/loading-data.rst similarity index 100% rename from doc/loading_data.rst rename to doc/loading-data.rst diff --git a/doc/paster.rst b/doc/paster.rst index 686212da29b..6d607af3756 100644 --- a/doc/paster.rst +++ b/doc/paster.rst @@ -1,6 +1,6 @@ -================= -Common CKAN Tasks -================= +=============================== +Common CKAN Administrator Tasks +=============================== The majority of common CKAN administration tasks are carried out using the **paster** script. @@ -20,8 +20,8 @@ At its simplest, paster commands can be thought of like this:: But there are various extra elements to the commandline that usually need adding. We shall build them up: -#. Enabling CKAN commands -========================= +Enabling CKAN commands +====================== Paster is used for many things aside from CKAN. You usually need to tell paster that you want to enable the CKAN commands:: @@ -31,8 +31,8 @@ You know you need to do this if you get the error ``Command 'user' not known`` f (Alternatively, CKAN commands are enabled by default if your current directory is the CKAN source directory) -#. Pointing to your CKAN config -=============================== +Pointing to your CKAN config +============================ Paster needs to know where your CKAN config file is (so it knows which database and search index to deal with etc.):: @@ -46,8 +46,8 @@ For example, to initialise a database:: paster --plugin=ckan db init --config=/etc/ckan/std/std.ini -#. Virtual environments -======================= +Virtual environments +==================== You often need to run paster within your CKAN virtual environment (pyenv). If CKAN was installed as 'source' then you can activate it as usual before running the paster command:: @@ -59,8 +59,8 @@ The alternative, which also suits a CKAN 'package' install, is to simply give th /var/lib/ckan/std/pyenv/bin/paster --plugin=ckan db init --config=/etc/ckan/std/std.ini -#. Running Paster on a deployment -================================= +Running Paster on a deployment +============================== If CKAN is deployed with Apache on this machine, then you should run paster as the same user, which is usually ``www-data``. This is because paster will write to the same CKAN logfile as the Apache process and file permissions need to match. @@ -198,7 +198,7 @@ To load it in again, you first have to clean the database of existing data (be c .. warning: The pg_dump file is a complete backup of the database in plain text, and includes API keys and other user data which may be regarded as private. So keep it secure, like your database server. -The pg_dump file notes which PostgreSQL user 'owns' the data on the server. Because the PostgreSQL user (by default) is identified as the current Linux user, and this is setup to be ``ckanINSTANCE`` where ``INSTANCE`` is the name of the CKAN instance. This means if you want to restore the pg_dump as another CKAN instance name (often needed if you move it to another server) then you will need to change the database owner - see :doc:`editing_the_database_ownership`. +The pg_dump file notes which PostgreSQL user 'owns' the data on the server. Because the PostgreSQL user (by default) is identified as the current Linux user, and this is setup to be ``ckanINSTANCE`` where ``INSTANCE`` is the name of the CKAN instance. This means if you want to restore the pg_dump as another CKAN instance name (often needed if you move it to another server) then you will need to change the database owner - see :doc:`howto-editing--database-ownership`. Upgrade migration ~~~~~~~~~~~~~~~~~ From 86d2bc53a74455dbccade0a07fc815d8f40ada72 Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Fri, 16 Mar 2012 09:57:32 +0000 Subject: [PATCH 6/6] [#2226,doc][xs]: put all files back at base level (no point putting things in directories and much simpler). --- doc/{dev => }/buildbot.rst | 0 doc/{storage => }/datastore.rst | 0 doc/{storage => }/filestore.rst | 0 doc/index.rst | 6 +++--- 4 files changed, 3 insertions(+), 3 deletions(-) rename doc/{dev => }/buildbot.rst (100%) rename doc/{storage => }/datastore.rst (100%) rename doc/{storage => }/filestore.rst (100%) diff --git a/doc/dev/buildbot.rst b/doc/buildbot.rst similarity index 100% rename from doc/dev/buildbot.rst rename to doc/buildbot.rst diff --git a/doc/storage/datastore.rst b/doc/datastore.rst similarity index 100% rename from doc/storage/datastore.rst rename to doc/datastore.rst diff --git a/doc/storage/filestore.rst b/doc/filestore.rst similarity index 100% rename from doc/storage/filestore.rst rename to doc/filestore.rst diff --git a/doc/index.rst b/doc/index.rst index 1760bb68d46..61ad018da4f 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -41,8 +41,8 @@ Publishing Datasets loading-data authorization - storage/filestore - storage/datastore + filestore + datastore The CKAN API ============ @@ -72,7 +72,7 @@ For CKAN Developers i18n test - dev/buildbot + buildbot Other material ==============