Merge pull request #124 from freedomofpress/wheels-via-git-lfs

Converts wheel storage from S3 to git-lfs

.circleci/config.yml

@@ -5,11 +5,6 @@ common-steps:
       name: Install Debian packaging dependencies
       command: make install-deps
 
-  - &fetchwheels
-    run:
-      name: Download wheels and sources
-      command: make fetch-wheels
-
   - &getlatestreleasedversion
     run:
       name: Get latest released version of the project
@@ -104,18 +99,6 @@ common-steps:
         make $PKG_NAME
         ls ~/debbuild/packaging/*.deb
 
-  - &installgitlfs
-    run:
-      name: Install Git LFS.
-      command: |
-        export GIT_LFS_VERSION=2.7.2
-        export GIT_LFS_CHECKSUM=89f5aa2c29800bbb71f5d4550edd69c5f83e3ee9e30f770446436dd7f4ef1d4c
-        wget https://github.com/git-lfs/git-lfs/releases/download/v$GIT_LFS_VERSION/git-lfs-linux-amd64-v$GIT_LFS_VERSION.tar.gz
-        sha256sum git-lfs-linux-amd64-v$GIT_LFS_VERSION.tar.gz | grep $GIT_LFS_CHECKSUM
-        tar xzf git-lfs-linux-amd64-v$GIT_LFS_VERSION.tar.gz
-        sudo mv git-lfs /usr/local/bin/git-lfs
-        git lfs install
-
   - &addsshkeys
     add_ssh_keys:
       fingerprints:
@@ -190,7 +173,6 @@ jobs:
     steps:
       - checkout
       - *installdeps
-      - *fetchwheels
       - *clonesecuredroplog
       - *getlatestreleasedversion
       - *makesourcetarball
@@ -202,13 +184,11 @@ jobs:
     steps:
       - checkout
       - *installdeps
-      - *fetchwheels
       - *clonesecuredroplog
       - *getnightlyversion
       - *makesourcetarball
       - *updatedebianchangelog
       - *builddebianpackage
-      - *installgitlfs
       - *addsshkeys
       - *commitworkstationdebs
 
@@ -218,7 +198,6 @@ jobs:
     steps:
       - checkout
       - *installdeps
-      - *fetchwheels
       - *clonesecuredropclient
       - *getlatestreleasedversion
       - *makesourcetarball
@@ -230,13 +209,11 @@ jobs:
     steps:
       - checkout
       - *installdeps
-      - *fetchwheels
       - *clonesecuredropclient
       - *getnightlyversion
       - *makesourcetarball
       - *updatedebianchangelog
       - *builddebianpackage
-      - *installgitlfs
       - *addsshkeys
       - *commitworkstationdebs
 
@@ -246,7 +223,6 @@ jobs:
     steps:
       - checkout
       - *installdeps
-      - *fetchwheels
       - *clonesecuredropproxy
       - *getlatestreleasedversion
       - *makesourcetarball
@@ -258,13 +234,11 @@ jobs:
     steps:
       - checkout
       - *installdeps
-      - *fetchwheels
       - *clonesecuredropproxy
       - *getnightlyversion
       - *makesourcetarball
       - *updatedebianchangelog
       - *builddebianpackage
-      - *installgitlfs
       - *addsshkeys
       - *commitworkstationdebs
 
@@ -274,7 +248,6 @@ jobs:
     steps:
       - checkout
       - *installdeps
-      - *fetchwheels
       - *clonesecuredropexport
       - *getlatestreleasedversion
       - *makesourcetarball
@@ -286,13 +259,11 @@ jobs:
     steps:
       - checkout
       - *installdeps
-      - *fetchwheels
       - *clonesecuredropexport
       - *getnightlyversion
       - *makesourcetarball
       - *updatedebianchangelog
       - *builddebianpackage
-      - *installgitlfs
       - *addsshkeys
       - *commitworkstationdebs
 
@@ -302,7 +273,6 @@ jobs:
     steps:
       - checkout
       - *installdeps
-      - *fetchwheels
       - *setsvsdispname
       - *setmetapackageversion
       - *builddebianpackage
@@ -313,7 +283,6 @@ jobs:
     steps:
       - checkout
       - *installdeps
-      - *fetchwheels
       - *setsdgrsecname
       - *setmetapackageversion
       - *builddebianpackage

.gitattributes

@@ -0,0 +1,4 @@
+*.deb filter=lfs diff=lfs merge=lfs -text
+*.rpm filter=lfs diff=lfs merge=lfs -text
+*.tar.gz filter=lfs diff=lfs merge=lfs -text
+*.whl filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,3 +1,2 @@
 debhelper-build-stamp
 *.debhelper.log
-localwheels/

Makefile

@@ -1,13 +1,9 @@
 DEFAULT_GOAL: help
 
 .PHONY: wheel-urls
-wheel-urls: ## Creates download URLs from s3 bucket from sha256sums.txt file
+wheel-urls: ## Creates download URLs for PyPI mirror from sha256sums.txt file
 	./scripts/createdownloadurls.py > wheelsurls.txt
 
-.PHONY: fetch-wheels
-fetch-wheels: ## Downloads wheels and sources from the remote server
-	./scripts/fetch-wheels
-
 .PHONY: securedrop-proxy
 securedrop-proxy: ## Builds Debian package for securedrop-proxy code
 	PKG_NAME="securedrop-proxy" ./scripts/build-debianpackage
@@ -44,7 +40,7 @@ requirements: ## Creates requirements files for the Python projects
 	./scripts/update-requirements
 
 .PHONY: build-wheels
-build-wheels: fetch-wheels ## Builds the wheels and adds them to the localwheels directory
+build-wheels: ## Builds the wheels and adds them to the localwheels directory
 	./scripts/verify-sha256sum-signature
 	./scripts/build-sync-wheels -p ${PKG_DIR}
 	./scripts/sync-sha256sums

README.md

@@ -20,12 +20,14 @@ In a Debian AppVM in Qubes:
 
 ```
 make install-deps
-make fetch-wheels
 ```
 
 **Note:** either run `make install-deps` each time you start your debian packaging AppVM, or make
 sure that you install them into the template for your debian packaging AppVM.
 
+The install target will configure [git-lfs](https://git-lfs.github.com/), used for storing
+binary wheel files.
+
 ## Updating Python wheels
 
 Maintainers of `securedrop-client` and `securedrop-proxy` must ensure that
@@ -36,11 +38,7 @@ If new dependencies were added in the `requirements.txt` of that
 repo that are not in the FPF PyPI mirror, then the maintainer needs
 to do the following (we are taking `securedrop-client` project as example):
 
-### 1. Sync the wheels locally
-
-Sync all of the latest wheels `make fetch-wheels`
-
-### 2. Create updated build-requirements.txt for the project
+### 1. Create updated build-requirements.txt for the project
 
 From the `securedrop-debian-packaging` directory,
 
@@ -59,8 +57,8 @@ pytest==3.10.1
 
 Please build the wheel by using the following command.
 	PKG_DIR=/home/user/code/securedrop-client make build-wheels
-Then sync the newly built wheels and sources to the s3 bucket.
-Also update the index HTML files accordingly and sync to s3.
+Then add the newly built wheels and sources to ./localwheels/.
+Also update the index HTML files accordingly commit your changes.
 After these steps, please rerun the command again.
 ```
 
@@ -84,18 +82,18 @@ python3 setup.py sdist
 ```
 
 
-### 3. Sync the localwheels directory back to the s3 bucket. (if only any update of wheels)
+### 2. Commit changes to the localwheels directory (if only any update of wheels)
 
 This has to be manual step for security reasons. In future all of these wheel
 building steps should be done by a different system, not with the devloper's
 laptop.
 
 ```
-cd localwheels/
-aws s3 sync . s3://dev-bin.ops.securedrop.org/localwheels/
+git add localwheels/
+git commit
 ```
 
-### 4. Update the index files for the bucket (required for Debian builds)
+### 3. Update the index files for the bucket (required for Debian builds)
 
 If there is any completely new Python package (source/wheel), then only we will have to update our index.
 
@@ -110,13 +108,18 @@ If there is a new package, then update the main index.
 ./scripts/updateindex.py
 ```
 
-Finally sync the index.
+Finally, submit a PR containing the new wheels and updated files.
+If you wish to test the new wheels in a local build before submitting a PR,
+or as part of PR review, you can do so by:
 
 ```
-cd simple/
-s3 sync . s3://dev-bin.ops.securedrop.org/simple/
+python3 -m http.server # serve local wheels via HTTP
+vim $PKG_NAME/debian/rules # edit index URL to http://localhost:8000/simple
 ```
 
+Then run e.g. `PKG_VERSION=0.0.11 make securedrop-client`, and you'll see the GET
+requests in the console running the HTTP server.
+
 ## Make a release
 
 Summarizing release manager steps: