Fix #2642: Improve exporting files to mozilla-central, avoid duplicat…

…e L10n files (#2645)

The m-c build system protects against duplicate files, so we need to avoid those. Additionally, this patch makes it
so that we don't export empty l10n files as we just fallback to en-US.

Makefile

@@ -90,17 +90,6 @@ build/%.html: %.html
 .PHONY: addon
 addon: npm set_backend set_sentry addon/webextension/manifest.json addon/install.rdf addon_locales addon/webextension/build/shot.js addon/webextension/build/inlineSelectionCss.js addon/webextension/build/raven.js addon/webextension/build/defaultSentryDsn.js addon/webextension/build/onboardingCss.js addon/webextension/build/onboardingHtml.js
 
-EXPORT_MC_LOCATION := $(shell echo $${EXPORT_MC_LOCATION-../gecko})
-GIT_EXPORT_DIR := $(EXPORT_MC_LOCATION)/browser/extensions/screenshots
-DIST_EXPORT_DIR := addon
-TEST_EXPORT_DIR := test/addon
-
-ifeq ($(shell uname -s),Linux)
-  FIND_COMMAND := find $(GIT_EXPORT_DIR) -regextype posix-extended
-else
-  FIND_COMMAND := find -E $(GIT_EXPORT_DIR)
-endif
-
 $(VENV): bin/require.pip
 	virtualenv -p python2.7 $(VENV)
 	. $(VENV)/bin/activate && pip install -r bin/require.pip
@@ -109,16 +98,6 @@ $(VENV): bin/require.pip
 flake8: $(VENV)
 	$(VENV)/bin/flake8 .
 
-.PHONY: export_addon
-export_addon: addon
-	$(FIND_COMMAND) -type f ! -regex \
-		'.*/(moz.build|README.txt|.gitignore|manifest.ini)' -delete
-	$(RSYNC) $(DIST_EXPORT_DIR)/* $(GIT_EXPORT_DIR)
-	@mkdir -p $(GIT_EXPORT_DIR)/test/browser
-	$(RSYNC) $(TEST_EXPORT_DIR)/* $(GIT_EXPORT_DIR)/test/browser
-	rm -f $(GIT_EXPORT_DIR)/README.md $(GIT_EXPORT_DIR)/install.rdf.template
-	$(VENV)/bin/python bin/update_mozbuild.py
-
 .PHONY: zip
 zip: addon
 	# FIXME: should remove web-ext-artifacts/*.zip first

bin/export_mc.py

@@ -4,7 +4,9 @@
 import os
 import subprocess
 import sys
+import filecmp
 from git import Repo
+from shutil import copyfile
 
 DEFAULTS = {
     "local": "http://localhost:10080",
@@ -13,6 +15,17 @@
     "prod": "https://screenshots.firefox.com"
 }
 
+FILES_TO_NOT_REMOVE = [
+    'moz.build',
+    'README.txt'
+]
+
+FILES_TO_SKIP_COPY = [
+    'README.md',
+    'install.rdf.template',
+    'manifest.json.template'
+]
+
 
 def runProcess(cmd, cwd, errorMessage):
     print "runProcess ", cmd
@@ -72,12 +85,59 @@ def createCommit(self, subDir, commitMessage):
                        "Failed to add files for commit: %s")
 
 
+def exportFilesToMC(repoDir, mcRepoLoc):
+    print "Exporting files"
+
+    # Remove the existing files, except for ones we need to keep.
+    for root, dirs, files in os.walk(mcRepoLoc):
+        for file in files:
+            if file not in FILES_TO_NOT_REMOVE:
+                os.remove(os.path.join(root, file))
+
+    addonDir = os.path.join(repoDir, "addon")
+    testDir = os.path.join(repoDir, "test", "addon")
+    enUSFile = os.path.join(addonDir, "webextension", "_locales", "en_US", "messages.json")
+
+    # Export the main add-on files.
+    for root, dirs, files in os.walk(addonDir):
+        relativePath = os.path.relpath(root, addonDir)
+
+        for file in files:
+            filePath = os.path.join(root, file)
+            if file == "messages.json" and os.path.getsize(filePath) == 2:
+                print "Skipping empty locale file: %s" % filePath
+            elif file == "messages.json" and 'en_US' not in root and filecmp.cmp(enUSFile, filePath):
+                print "Skipping same as en-US locale file: %s" % filePath
+            elif file not in FILES_TO_SKIP_COPY:
+                copyfile(filePath,
+                         os.path.join(mcRepoLoc, relativePath, file))
+
+        for dir in dirs:
+            dirPath = os.path.join(mcRepoLoc, relativePath, dir)
+            if not os.path.exists(dirPath):
+                os.mkdir(dirPath, 0755)
+
+    # Copy the test files.
+    mc_test_loc = os.path.join(mcRepoLoc, "test", "browser")
+    if not os.path.exists(mc_test_loc):
+        os.makedirs(mc_test_loc, 0755)
+
+    for root, dirs, files in os.walk(testDir):
+        for file in files:
+            copyfile(os.path.join(root, file),
+                     os.path.join(mc_test_loc, os.path.relpath(root, testDir), file))
+
+    # Finally, update the moz.build file.
+    runProcess([".venv/bin/python", "bin/update_mozbuild.py"], repoDir,
+               "Failed to run update_mozbuild.py %s")
+
+
 def exportToMozillaCentral(server, repoDir, mcRepoPath, mcSubDir, mcBranch,
                            noSwitchBranch, mcBaseCommit, commitMessage):
     print "Exporting to m-c"
 
-    os.environ["EXPORT_MC_LOCATION"] = mcRepoPath
     os.environ["SCREENSHOTS_BACKEND"] = DEFAULTS[server]
+    os.environ["SCREENSHOTS_MINOR_VERSION"] = "0"
 
     repo = RepoHandler(mcRepoPath)
 
@@ -90,7 +150,9 @@ def exportToMozillaCentral(server, repoDir, mcRepoPath, mcSubDir, mcBranch,
 
     runProcess(['make', 'clean'], repoDir, "Failed to make clean: %s")
 
-    runProcess(['make', 'export_addon'], repoDir, "Failed to make export_mc: %s")
+    runProcess(['make', 'addon'], repoDir, "Failed to make addon: %s")
+
+    exportFilesToMC(repoDir, os.path.join(mcRepoPath, "browser", "extensions", "screenshots"))
 
     repo.createCommit(mcSubDir, commitMessage)
 
@@ -148,15 +210,14 @@ def main(server, mcRepoPath, mcSubDir, mcBranch, noSwitchBranch,
                         default="prod",
                         help="[local|dev|stage|prod]: Which server the code should use.")
     parser.add_argument("--mozilla-central-repo",
-                        default=os.environ["EXPORT_MC_LOCATION"],
-                        metavar="../gecko-dev",
+                        default=(os.getenv("EXPORT_MC_LOCATION") or "../gecko-dev"),
+                        metavar=(os.getenv("EXPORT_MC_LOCATION") or "../gecko-dev"),
                         help="A gecko directory reference to mozilla-central, can also "
                              "be specified via EXPORT_MC_LOCATION environment variable")
     parser.add_argument("--mozilla-central-subdir",
                         default="browser/extensions/screenshots/",
                         help="Where the extension is located in mozilla-central.")
     parser.add_argument("-b", "--branch",
-                        required=True,
                         help="The branch/bookmark name to use for the export.")
     parser.add_argument("--no-switch-branch",
                         action="store_true",