Skip to content
This repository

Remove the thirdparty.zip malarkey. Potentially significant performance win, #35

Open
wants to merge 1 commit into from

1 participant

James Polley
James Polley
Owner

significant decrease in complexity of maintaining the app, at a
cost of bumping the number of files to be uploaded from just over 200 to
just over 300.

If we ever need this back, it's in the git history and we can pull it back out.

James Polley jamezpolley Remove the thirdparty.zip malarkey. Potentially significant performance
win, significant decrease in complexity of maintaining the app, at a
cost of bumping the number of files to be uploaded from just over 200 to
just over 300.
7350313
James Polley
Owner

Exact numbers:

the version7 branch:
Cloning 40 static files.
Cloning 144 application files

Current master branch:
Cloning 40 static files.
Cloning 148 application files.

After this patch:
Cloning 40 static files.
Cloning 277 application files.

The limit is 3k files. We have a long way to go before the zip functionality helps us in any way at all.

James Polley
Owner

After talking to other people yesterday, it seems as though the performance gain here is probably not as significant as I had thought.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Showing 1 unique commit by 1 author.

Jun 11, 2011
James Polley jamezpolley Remove the thirdparty.zip malarkey. Potentially significant performance
win, significant decrease in complexity of maintaining the app, at a
cost of bumping the number of files to be uploaded from just over 200 to
just over 300.
7350313
This page is out of date. Refresh to see the latest.

Showing 4 changed files with 10 additions and 52 deletions. Show diff stats Hide diff stats

  1. +2 37 Makefile
  2. +1 2  app.yaml
  3. +1 7 config.py
  4. +6 6 third_party.paths
39 Makefile
@@ -103,50 +103,15 @@ TPT=third_party.zip.d.tmp
103 103 third_party:
104 104 make -C third_party
105 105
106   -third_party.zip.d: third_party third_party.paths
107   - @echo "Generating a third_party.zip dependency file."
108   - @echo "THIRD_PARTY_files=\\" > $(TPT).files
109   - @echo 'third_party.zip: third_party.zip.d $$(THIRD_PARTY_here)' > $(TPT).target
110   - @echo ' @echo Changed files: $$?' >> $(TPT).target
111   - @cd third_party; \
112   - cat ../third_party.paths | grep ^third_party.zip | sed -e's-third_party.zip/--' | (while read LINE; do \
113   - PREFIX=$${LINE%% *}; \
114   - for SUFFIX in $${LINE#* }; do \
115   - find $$PREFIX/$$SUFFIX $(FINDARGS) >> ../$(TPT).files; \
116   - echo " cd $(CURDIR)/third_party/$$PREFIX; zip -r $(CURDIR)/third_party.zip \\" >> ../$(TPT).target; \
117   - (cd $$PREFIX; find $$SUFFIX $(FINDARGS)) >> ../$(TPT).target; \
118   - echo " " >> ../$(TPT).target; \
119   - done; \
120   - done)
121   - @echo '' >> $(TPT).files
122   - @echo 'THIRD_PARTY_here=$$(addprefix third_party/,$$(THIRD_PARTY_files)) ' >> $(TPT).files
123   - @echo '' >> $(TPT).files
124   -
125   - @echo '' > $(TPT)
126   - @cat $(TPT).files >> $(TPT)
127   - @cat $(TPT).target >> $(TPT)
128   -
129   - @if [ ! -e $@ ]; then touch $@; fi
130   - @if [ `${MD5SUM} $@ | sed -e's/ .*//'` != `${MD5SUM} $(TPT) | sed -e's/ .*//'` ]; then \
131   - echo "third_party.zip.d has changed!"; \
132   - mv $(TPT) $@; \
133   - else \
134   - echo "third_party.zip.d has not changed!"; \
135   - fi
136   - @rm $(TPT)*
137   -
138   -
139   -include third_party.zip.d
140   -
141 106 ###############################################################################
142 107 ###############################################################################
143 108
144 109 upload: update
145 110 deploy: update
146   -update: third_party.zip
  111 +update:
147 112 ${APPENGINE_SDK}/appcfg.py update .
148 113
149   -serve: third_party.zip
  114 +serve:
150 115 ${APPENGINE_SDK}/dev_appserver.py -a 0.0.0.0 -d --enable_sendmail .
151 116
152 117 clean:
3  app.yaml
@@ -40,10 +40,9 @@ handlers:
40 40 script: handler.py
41 41
42 42 skip_files:
43   -- third_party/((?!(aeoid|python-gae-pytz)).*)/
44   -- third_party/python-gae-pytz/(?!pytz)
45 43 - third_party/aeoid/demo
46 44 - third_party/mkzip
  45 +- third_party/python-markdown/tests
47 46 - (.*/)*.git/
48 47 - .*/\..*
49 48 - .*/README
8 config.py
@@ -34,7 +34,6 @@ def sys_path_insert(ipath):
34 34 if ipath not in sys.path:
35 35 sys.path.insert(0, ipath)
36 36
37   -
38 37 def setup_django():
39 38 """Setup the django settings."""
40 39 os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
@@ -52,10 +51,8 @@ def setup(host=None):
52 51 HOST = host
53 52
54 53 # Add our extra modules to sys.path
55   - sys_path_insert('third_party.zip')
  54 + sys_path_insert('third_party')
56 55 for ipath in getpaths():
57   - if 'third_party.zip' in ipath:
58   - continue
59 56 sys_path_insert(ipath)
60 57
61 58 setup_django()
@@ -70,9 +67,6 @@ def lint_setup():
70 67 print "APPENGINE_SDK at ", os.environ["APPENGINE_SDK"]
71 68 sys_path_insert(os.environ["APPENGINE_SDK"])
72 69
73   - for ipath in getpaths():
74   - sys_path_insert(ipath.replace('.zip', ''))
75   -
76 70 setup_django()
77 71
78 72 if __name__ == "__main__":
12 third_party.paths
... ... @@ -1,9 +1,9 @@
1 1 third_party/aeoid
2 2 third_party/python-gae-pytz
3 3 # Directory to add to sys.path, things to include in third_party.zip
4   -third_party.zip/python-dateutil dateutil
5   -third_party.zip/python-datetime-tz datetime_tz.py pytz_abbr.py
6   -third_party.zip/python-markdown markdown
7   -third_party.zip/python-markdown-extensions markdown
8   -third_party.zip/python-pyrss2gen PyRSS2Gen.py
9   -third_party.zip/python-vobject vobject
  4 +third_party/python-dateutil dateutil
  5 +third_party/python-datetime-tz datetime_tz.py pytz_abbr.py
  6 +third_party/python-markdown markdown
  7 +third_party/python-markdown-extensions markdown
  8 +third_party/python-pyrss2gen PyRSS2Gen.py
  9 +third_party/python-vobject vobject

Tip: You can add notes to lines in a file. Hover to the left of a line to make a note

Something went wrong with that request. Please try again.