Merge 1746844 into ad64151

.travis.yml

@@ -6,9 +6,6 @@ python:
   - '3.8'
 env:
   - TOXENV=quality
-  - TOXENV=django22
-  - TOXENV=django30
-  - TOXENV=django31
   - TOXENV=django32
 cache:
   directories:

Makefile

@@ -1,4 +1,4 @@
-.PHONY: clean test upgrade test requirements quality-python
+.PHONY: clean piptools test upgrade test requirements quality-python
 # Generates a help message. Borrowed from https://github.com/pydanny/cookiecutter-djangopackage.
 help: ## display this help message
 	@echo "Please use \`make <target>\` where <target> is one of"
@@ -14,14 +14,34 @@ clean: ## delete generated byte code and coverage reports
 	rm -rf assets
 	rm -rf pii_report
 
+piptools: ## install pinned version of pip-compile and pip-sync
+	pip install -r requirements/pip_tools.txt
+
+## The following defines a string constant that contains a multi-line comment.
+define COMMON_CONSTRAINTS_TEMP_COMMENT
+# This is a temporary solution to override the real common_constraints.txt\n# In edx-lint, until the pyjwt constraint in edx-lint has been removed.\n# See BOM-2721 for more details.\n# Below is the copied and edited version of common_constraints\n
+endef
+
+COMMON_CONSTRAINTS_TXT=requirements/common_constraints.txt
+.PHONY: $(COMMON_CONSTRAINTS_TXT)
+$(COMMON_CONSTRAINTS_TXT):
+	wget -O "$(@)" https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt || touch "$(@)"
+	echo "$(COMMON_CONSTRAINTS_TEMP_COMMENT)" | cat - $(@) > temp && mv temp $(@)
+
 upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade
-upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
-	pip install -q -r requirements/pip_tools.txt
+upgrade: piptools $(COMMON_CONSTRAINTS_TXT)## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
+
+	sed 's/Django<2.3//g' requirements/common_constraints.txt > requirements/common_constraints.tmp
+	mv requirements/common_constraints.tmp requirements/common_constraints.txt
+
+	pip-compile --rebuild --allow-unsafe -o requirements/pip.txt requirements/pip.in
 	pip-compile --upgrade -o requirements/pip_tools.txt requirements/pip_tools.in
 	pip-compile --upgrade -o requirements/base.txt requirements/base.in
 	pip-compile --upgrade -o requirements/test.txt requirements/test.in
 	pip-compile --upgrade -o requirements/tox.txt requirements/tox.in
 	pip-compile --upgrade -o requirements/travis.txt requirements/travis.in
+	pip-compile --upgrade -o requirements/dev.txt requirements/dev.in
+
 	# Let tox control the Django version for tests
 	grep -e "^django==" requirements/base.txt > requirements/django.txt
 	sed '/^[dD]jango==/d' requirements/test.txt > requirements/test.tmp
@@ -30,7 +50,7 @@ upgrade: ## update the requirements/*.txt files with the latest packages satisfy
 requirements: ## install development environment requirements
 	pip install -q -r requirements/pip_tools.txt
 	pip install -qr requirements/base.txt --exists-action w
-	pip-sync requirements/base.txt requirements/constraints.txt requirements/test.txt requirements/travis.txt
+	pip-sync requirements/dev.txt
 
 test-python: clean ## run tests using pytest and generate coverage report
 	$(TOX)pytest

i18n/config.py

@@ -58,7 +58,7 @@ def read_config(self, filename):
         Returns data found in config file (as dict), or raises exception if file not found
         """
         if not os.path.exists(filename):
-            raise Exception("Configuration file cannot be found: %s" % filename)
+            raise Exception(f"Configuration file cannot be found: {filename}")
         with open(filename, encoding='UTF-8') as stream:
             return yaml.safe_load(stream)
 

i18n/converter.py

@@ -56,7 +56,7 @@ def detag_string(self, string):
            list: list of the removed tags ('<BR>', '<I>', '</I>')
         """
         counter = itertools.count(0)
-        count = lambda m: '<%s>' % next(counter)
+        count = lambda m: f'<{next(counter)}>'
         tags = self.tag_pattern.findall(string)
         tags = [''.join(tag) for tag in tags]
         (new, nfound) = self.tag_pattern.subn(count, string)
@@ -67,7 +67,7 @@ def detag_string(self, string):
     def retag_string(self, string, tags):
         """substitutes each tag back into string, into occurrences of <0>, <1> etc"""
         for i, tag in enumerate(tags):
-            bracketed = '<%s>' % i
+            bracketed = f'<{i}>'
             string = re.sub(bracketed, tag, string, 1)
         return string
 

i18n/dummy.py

@@ -191,7 +191,7 @@ def make_dummy(filename, locale, converter):
     in :param locale: containing a dummy translation.
     """
     if not Path(filename).exists():
-        raise OSError('File does not exist: %r' % filename)
+        raise OSError(f'File does not exist: {filename}')
     pofile = polib.pofile(filename)
     for msg in pofile:
         # Some strings are actually formatting strings, don't dummy-ify them,

i18n/execute.py

@@ -29,9 +29,9 @@ def call(command, working_directory=config.BASE_DIR):
 
     """
     LOG.info(command)
-    proc = sp.Popen(command, stdout=sp.PIPE, stderr=sp.PIPE, cwd=working_directory, shell=True)
-    out, err = proc.communicate()
-    return (out, err)
+    with sp.Popen(command, stdout=sp.PIPE, stderr=sp.PIPE, cwd=working_directory, shell=True) as proc:
+        out, err = proc.communicate()
+        return out, err
 
 
 def remove_file(filename, verbose=True):

i18n/extract.py

@@ -32,7 +32,7 @@
 
 EDX_MARKER = "edX translation file"
 LOG = logging.getLogger(__name__)
-DEVNULL = open(os.devnull, 'wb')
+DEVNULL = open(os.devnull, 'wb')    # pylint: disable=consider-using-with
 
 
 class Extract(Runner):

i18n/generate.py

@@ -25,7 +25,7 @@
 from i18n.execute import execute
 
 LOG = logging.getLogger(__name__)
-DEVNULL = open(os.devnull, "wb")
+DEVNULL = open(os.devnull, "wb")        # pylint: disable=consider-using-with
 DUPLICATE_ENTRY_PATTERN = re.compile('#-#-#-#-#.*#-#-#-#-#')
 
 
@@ -111,10 +111,10 @@ def clean_pofile(pofile_path):
             # Remove fuzzy from flags
             entry.flags = [f for f in entry.flags if f != 'fuzzy']
             # Save a warning message
-            dup_msg = 'Multiple translations found for single string.\n\tString "{}"\n\tPresent in files {}'.format(
-                entry.msgid,
-                [f for (f, __) in entry.occurrences]
-            )
+            occurrences = [f for (f, __) in entry.occurrences]
+            dup_msg = (f'Multiple translations found for single string.\n\t'
+                       f'String "{entry.msgid}"\n\tPresent in files {occurrences}'
+                       )
             duplicate_entries.append((dup_msg, entry.msgstr))
 
             # Pick the first entry
@@ -128,10 +128,9 @@ def clean_pofile(pofile_path):
                     # Raise error if there's new lines starting or ending the id string.
                     if entry.msgid.startswith('\n') or entry.msgid.endswith('\n'):
                         raise ValueError(
-                            '{} starts or ends with a new line character, which is not allowed. '
-                            'Please fix before continuing. Source string is found in {}'.format(
-                                entry.msgid, entry.occurrences
-                            ).encode('utf-8')
+                            f'{entry.msgid} starts or ends with a new line character, which is not allowed. '
+                            'Please fix before continuing. Source string is found in {entry.occurrences}'
+                            .encode('utf-8')
                         )
                     break
 

i18n/main.py

@@ -20,7 +20,7 @@ def get_valid_commands():
     for modname in modules:
         if modname == 'main':
             continue
-        mod = importlib.import_module('i18n.%s' % modname)
+        mod = importlib.import_module(f'i18n.{modname}')
         if hasattr(mod, 'main'):
             commands.append(modname)
     return commands
@@ -35,7 +35,7 @@ def error_message():
     """
     sys.stderr.write('valid commands:\n')
     for cmd in get_valid_commands():
-        sys.stderr.write('\t%s\n' % cmd)
+        sys.stderr.write(f'\t{cmd}\n')
     return -1
 
 
@@ -52,7 +52,7 @@ def main():
         return error_message()
 
     try:
-        module = importlib.import_module('i18n.%s' % command)
+        module = importlib.import_module(f'i18n.{command}')
         module.main.args = sys.argv[2:]
     except (ImportError, AttributeError):
         return error_message()

i18n/validate.py

@@ -75,8 +75,8 @@ def msgfmt_check_po_file(locale_dir, filename):
     rfile = os.path.relpath(filename, locale_dir)
     out, err = call(f'msgfmt -c -o /dev/null {rfile}', working_directory=locale_dir)
     if err:
-        log.info('\n{}'.format(out.decode('utf8')))
-        log.warning('\n{}'.format(err.decode('utf8')))
+        log.info('\n%s', out.decode('utf8'))
+        log.warning('\n%s', err.decode('utf8'))
         found_problems = True
 
     return found_problems
@@ -110,7 +110,7 @@ def astral(msg):
     # By encoding as utf32, and decoding DWORDS, we can get at the real code
     # points.
     utf32 = msg.encode("utf32")[4:]         # [4:] to drop the bom
-    code_points = struct.unpack("%dI" % (len(utf32) / 4), utf32)
+    code_points = struct.unpack('%dI' % (len(utf32) / 4), utf32)
     return any(cp > 0xFFFF for cp in code_points)
 
 
@@ -191,11 +191,11 @@ def report_problems(filename, problems):
     with codecs.open(problem_file, "w", encoding="utf8") as prob_file:
         for problem in problems:
             desc, msgid = problem[:2]
-            prob_file.write("{}\n{}\n".format(desc, id_filler.fill(msgid)))
-            info = "{}\n{}\n".format(desc, id_filler.fill(msgid))
+            prob_file.write(f"{desc}\n{id_filler.fill(msgid)}\n")
+            info = f"{desc}\n{id_filler.fill(msgid)}\n"
             for translation in problem[2:]:
-                prob_file.write("{}\n".format(tx_filler.fill(translation)))
-                info += "{}\n".format(tx_filler.fill(translation))
+                prob_file.write(f"{tx_filler.fill(translation)}\n")
+                info += f"{tx_filler.fill(translation)}\n"
             log.info(info)
             prob_file.write("\n")
 

pylintrc

@@ -290,6 +290,7 @@ disable =
 	unused-wildcard-import,
 	global-statement,
 	no-else-return,
+	consider-using-f-string,
 	# These are disabled by pylint by default
 	apply-builtin,
 	backtick,

requirements/base.txt

@@ -1,13 +1,22 @@
 #
-# This file is autogenerated by pip-compile
+# This file is autogenerated by pip-compile with python 3.8
 # To update, run:
 #
 #    make upgrade
 #
-django==2.2.20            # via -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt, -r requirements/base.in
-path==15.1.2              # via -r requirements/base.in
-polib==1.1.1              # via -r requirements/base.in
-pytz==2021.1              # via django
-pyyaml==5.4.1             # via -r requirements/base.in
-six==1.15.0               # via -r requirements/base.in
-sqlparse==0.4.1           # via django
+asgiref==3.4.1
+    # via django
+django==3.2.7
+    # via -r requirements/base.in
+path==16.2.0
+    # via -r requirements/base.in
+polib==1.1.1
+    # via -r requirements/base.in
+pytz==2021.3
+    # via django
+pyyaml==5.4.1
+    # via -r requirements/base.in
+six==1.16.0
+    # via -r requirements/base.in
+sqlparse==0.4.2
+    # via django

requirements/common_constraints.txt

@@ -0,0 +1,48 @@
+# This is a temporary solution to override the real common_constraints.txt
+# In edx-lint, until the pyjwt constraint in edx-lint has been removed.
+# See BOM-2721 for more details.
+# Below is the copied and edited version of common_constraints
+
+# A central location for most common version constraints
+# (across edx repos) for pip-installation.
+#
+# Similar to other constraint files this file doesn't install any packages.
+# It specifies version constraints that will be applied if a package is needed.
+# When pinning something here, please provide an explanation of why it is a good
+# idea to pin this package across all edx repos, Ideally, link to other information
+# that will help people in the future to remove the pin when possible.
+# Writing an issue against the offending project and linking to it here is good.
+#
+# Note: Changes to this file will automatically be used by other repos, referencing
+#  this file from Github directly. It does not require packaging in edx-lint.
+
+
+# using LTS django version
+
+
+# latest version is causing e2e failures in edx-platform.
+# See pyjwt[crypto]<2.0.0 comment.
+drf-jwt<1.19.1
+
+# 4.0.0 requires pyjwt[crypto] 2.1.0. See pyjwt[crypto]<2.0.0 comment.
+edx-auth-backends<4.0.0
+
+# 7.0.0 requires pyjwt[crypto] 2.1.0. See pyjwt[crypto]<2.0.0 comment.
+edx-drf-extensions<7.0.0
+
+# PyJWT[crypto] 2.0.0 has a number of breaking changes that we are
+# actively working to fix. A number of the active constraints are all related
+# to this effort. Additionally, your IDA/service may also be affected directly
+# by these changes. You should not upgrade without knowing what you are doing.
+pyjwt[crypto]<2.0.0
+
+# 5.0.0+ of social-auth-app-django requires social-auth-core>=4.1.0
+social-auth-app-django<5.0.0
+
+# latest version requires PyJWT>=2.0.0 but drf-jwt requires PyJWT[crypto]<2.0.0,>=1.5.2.
+# See pyjwt[crypto]<2.0.0 comment.
+social-auth-core<4.0.3
+
+# elasticsearch>=7.14.0 includes breaking changes in it which caused issues in discovery upgrade process.
+# elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html
+elasticsearch<7.14.0

requirements/constraints.txt

@@ -9,12 +9,19 @@
 # linking to it here is good.
 
 # Common constraints for edx repos
--c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt
+-c common_constraints.txt
 
 # TODO: Many pinned dependencies should be unpinned and/or moved to this constraints file.
 
 # See https://openedx.atlassian.net/browse/BOM-2247 for details.
-pip-tools==5.3.0
+#pip-tools==5.3.0
 
 # mock version 4.0.0 drops support for python 3.5
 mock<4.0.0
+
+# coveralls package in travis.in constraints converage<6.0.
+# This only comes into play when upgrading travis.in, which happens after test.in
+# This constrain should be safe to remove after either:
+#  - coveralls package supports coverage>=6.0
+#  - travis.in is no longer necessary
+coverage<6.0

requirements/dev.in

@@ -0,0 +1,7 @@
+
+# Additional requirements for development of this application
+-c constraints.txt
+
+-r pip_tools.txt          # pip-tools and its dependencies, for managing requirements files
+-r travis.txt             # dependencies needed to setup testing environment on CI
+