Fully incorporate the code from Python 3.7.2 (#78)

This is a full port, following the recipe in update_process.md. I've also tried to keep the recipe up to date and improved the automation (see tools/script). I haven't cleaned up the commits. As of #77 there are a few tests that sanity-check this (though it's far from a full test suite), and they're run by Travis-CI and AppVeyor.

appveyor.yml

@@ -2,9 +2,6 @@ environment:
   matrix:
     # For Python versions available on Appveyor, see
     # http://www.appveyor.com/docs/installed-software#python
-    - PYTHON: "C:\\Python33"
-    - PYTHON: "C:\\Python33-x64"
-      DISTUTILS_USE_SDK: 1
     - PYTHON: "C:\\Python34"
     - PYTHON: "C:\\Python34-x64"
       DISTUTILS_USE_SDK: 1

ast27/Python/ast.c

@@ -1500,7 +1500,6 @@ ast_for_atom(struct compiling *c, const node *n)
     case STRING: {
         PyObject *kind, *str = parsestrplus(c, n);
         const char *raw, *s = STR(CHILD(n, 0));
-        int quote = Py_CHARMASK(*s);
         /* currently Python allows up to 2 string modifiers */
         char *ch, s_kind[3] = {0, 0, 0};
         ch = s_kind;
@@ -1519,7 +1518,7 @@ ast_for_atom(struct compiling *c, const node *n)
                 PyErr_Fetch(&type, &value, &tback);
                 errstr = PyObject_Str(value);
                 if (errstr) {
-                    char *s = "";
+                    const char *s = "";
                     char buf[128];
                     s = _PyUnicode_AsString(errstr);
                     PyOS_snprintf(buf, sizeof(buf), "(unicode error) %s", s);
@@ -2190,7 +2189,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
                 keyword_ty kw;
                 identifier key;
                 int k;
-                char *tmp;
+                const char *tmp;
 
                 /* CHILD(ch, 0) is test, but must be an identifier? */
                 e = ast_for_expr(c, CHILD(ch, 0));

ast3/Custom/typed_ast.c

@@ -1,6 +1,6 @@
 #include "Python.h"
 #include "Python-ast.h"
-#include "compile.h"
+#include "compile-ast3.h"
 #include "node.h"
 #include "grammar.h"
 #include "token.h"
@@ -222,10 +222,13 @@ string_object_to_c_ast(const char *s, PyObject *filename, int start,
     PyCompilerFlags localflags;
     perrdetail err;
     int iflags = PARSER_FLAGS(flags);
+    node *n;
 
-    node *n = Ta3Parser_ParseStringObject(s, filename,
-                                         &_Ta3Parser_Grammar, start, &err,
-                                         &iflags);
+    if (feature_version >= 7)
+        iflags |= PyPARSE_ASYNC_ALWAYS;
+    n = Ta3Parser_ParseStringObject(s, filename,
+                                    &_Ta3Parser_Grammar, start, &err,
+                                    &iflags);
     if (flags == NULL) {
         localflags.cf_flags = 0;
         flags = &localflags;

ast3/Grammar/Grammar

@@ -1,14 +1,7 @@
 # Grammar for Python
 
-# Note:  Changing the grammar specified in this file will most likely
-#        require corresponding changes in the parser module
-#        (../Modules/parsermodule.c).  If you can't make the changes to
-#        that module yourself, please co-ordinate the required changes
-#        with someone who can; ask around on python-dev for help.  Fred
-#        Drake <fdrake@acm.org> will probably be listening there.
-
 # NOTE WELL: You should also follow all the steps listed at
-# https://docs.python.org/devguide/grammar.html
+# https://devguide.python.org/grammar/
 
 # Start symbols for the grammar:
 #       single_input is a single interactive statement;
@@ -150,7 +143,8 @@ argument: ( test [comp_for] |
             '*' test )
 
 comp_iter: comp_for | comp_if
-comp_for: [ASYNC] 'for' exprlist 'in' or_test [comp_iter]
+sync_comp_for: 'for' exprlist 'in' or_test [comp_iter]
+comp_for: [ASYNC] sync_comp_for
 comp_if: 'if' test_nocond [comp_iter]
 
 # not used in grammar, but may appear in "node" passed from Parser to Compiler