F-strings!!!

ast35/Custom/typed_ast.c

@@ -242,6 +242,20 @@ string_object_to_c_ast(const char *s, PyObject *filename, int start,
     return mod;
 }
 
+// copy of PyParser_ASTFromString in Python/pythonrun.c
+mod_ty
+Ta35Parser_ASTFromString(const char *s, const char *filename_str, int start,
+                         PyCompilerFlags *flags, PyArena *arena) {
+    PyObject *filename;
+    mod_ty mod;
+    filename = PyUnicode_DecodeFSDefault(filename_str);
+    if (filename == NULL)
+        return NULL;
+    mod = string_object_to_c_ast(s, filename, start, flags, arena);
+    Py_DECREF(filename);
+    return mod;
+}
+
 // adapted from Py_CompileStringObject in Python/pythonrun.c
 static PyObject *
 string_object_to_py_ast(const char *str, PyObject *filename, int start,

ast35/Include/Python-ast.h

@@ -215,9 +215,10 @@ enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4,
                   SetComp_kind=9, DictComp_kind=10, GeneratorExp_kind=11,
                   Await_kind=12, Yield_kind=13, YieldFrom_kind=14,
                   Compare_kind=15, Call_kind=16, Num_kind=17, Str_kind=18,
-                  Bytes_kind=19, NameConstant_kind=20, Ellipsis_kind=21,
-                  Attribute_kind=22, Subscript_kind=23, Starred_kind=24,
-                  Name_kind=25, List_kind=26, Tuple_kind=27};
+                  FormattedValue_kind=19, JoinedStr_kind=20, Bytes_kind=21,
+                  NameConstant_kind=22, Ellipsis_kind=23, Attribute_kind=24,
+                  Subscript_kind=25, Starred_kind=26, Name_kind=27,
+                  List_kind=28, Tuple_kind=29};
 struct _expr {
     enum _expr_kind kind;
     union {
@@ -310,6 +311,16 @@ struct _expr {
             string s;
         } Str;
 
+        struct {
+            expr_ty value;
+            int conversion;
+            expr_ty format_spec;
+        } FormattedValue;
+
+        struct {
+            asdl_seq *values;
+        } JoinedStr;
+
         struct {
             bytes s;
         } Bytes;
@@ -574,6 +585,13 @@ expr_ty _Ta35_Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int
 expr_ty _Ta35_Num(object n, int lineno, int col_offset, PyArena *arena);
 #define Str(a0, a1, a2, a3) _Ta35_Str(a0, a1, a2, a3)
 expr_ty _Ta35_Str(string s, int lineno, int col_offset, PyArena *arena);
+#define FormattedValue(a0, a1, a2, a3, a4, a5) _Ta35_FormattedValue(a0, a1, a2, a3, a4, a5)
+expr_ty _Ta35_FormattedValue(expr_ty value, int conversion, expr_ty
+                             format_spec, int lineno, int col_offset, PyArena
+                             *arena);
+#define JoinedStr(a0, a1, a2, a3) _Ta35_JoinedStr(a0, a1, a2, a3)
+expr_ty _Ta35_JoinedStr(asdl_seq * values, int lineno, int col_offset, PyArena
+                        *arena);
 #define Bytes(a0, a1, a2, a3) _Ta35_Bytes(a0, a1, a2, a3)
 expr_ty _Ta35_Bytes(bytes s, int lineno, int col_offset, PyArena *arena);
 #define NameConstant(a0, a1, a2, a3) _Ta35_NameConstant(a0, a1, a2, a3)

ast35/Include/ast.h

@@ -15,6 +15,12 @@ PyAPI_FUNC(mod_ty) Ta35AST_FromNodeObject(
     PyCompilerFlags *flags,
     PyObject *filename,
     PyArena *arena);
+PyAPI_FUNC(mod_ty) Ta35Parser_ASTFromString(
+    const char *s,
+    const char *filename_str,
+    int start,
+    PyCompilerFlags *flags,
+    PyArena *arena);
 
 #ifdef __cplusplus
 }

ast35/Parser/Python.asdl

@@ -72,6 +72,8 @@ module Python
          | Call(expr func, expr* args, keyword* keywords)
          | Num(object n) -- a number as a PyObject.
          | Str(string s) -- need to specify raw, unicode, etc?
+         | FormattedValue(expr value, int? conversion, expr? format_spec)
+         | JoinedStr(expr* values)
          | Bytes(bytes s)
          | NameConstant(singleton value)
          | Ellipsis

ast35/Parser/tokenizer.c

@@ -1533,23 +1533,31 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
     /* Identifier (most frequent token!) */
     nonascii = 0;
     if (is_potential_identifier_start(c)) {
-        /* Process b"", r"", u"", br"" and rb"" */
-        int saw_b = 0, saw_r = 0, saw_u = 0;
+        /* Process b"", r"", u"", br"", rb"", rf"", and fr"" */
+        int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0;
         while (1) {
-            if (!(saw_b || saw_u) && (c == 'b' || c == 'B'))
+            if (!(saw_b || saw_u || saw_f) && (c == 'b' || c == 'B'))
                 saw_b = 1;
             /* Since this is a backwards compatibility support literal we don't
                want to support it in arbitrary order like byte literals. */
-            else if (!(saw_b || saw_u || saw_r) && (c == 'u' || c == 'U'))
+            else if (!(saw_b || saw_u || saw_r || saw_f)
+                     && (c == 'u'|| c == 'U')) {
                 saw_u = 1;
+            }
             /* ur"" and ru"" are not supported */
-            else if (!(saw_r || saw_u) && (c == 'r' || c == 'R'))
+            else if (!(saw_r || saw_u) && (c == 'r' || c == 'R')) {
                 saw_r = 1;
-            else
+            }
+            else if (!(saw_f || saw_b || saw_u) && (c == 'f' || c == 'F')) {
+                saw_f = 1;
+            }
+            else {
                 break;
+            }
             c = tok_nextc(tok);
-            if (c == '"' || c == '\'')
+            if (c == '"' || c == '\'') {
                 goto letter_quote;
+            }
         }
         while (is_potential_identifier_char(c)) {
             if (c >= 128)

ast35/Python/Python-ast.c

@@ -301,6 +301,18 @@ _Py_IDENTIFIER(s);
 static char *Str_fields[]={
     "s",
 };
+static PyTypeObject *FormattedValue_type;
+_Py_IDENTIFIER(conversion);
+_Py_IDENTIFIER(format_spec);
+static char *FormattedValue_fields[]={
+    "value",
+    "conversion",
+    "format_spec",
+};
+static PyTypeObject *JoinedStr_type;
+static char *JoinedStr_fields[]={
+    "values",
+};
 static PyTypeObject *Bytes_type;
 static char *Bytes_fields[]={
     "s",
@@ -941,6 +953,11 @@ static int init_types(void)
     if (!Num_type) return 0;
     Str_type = make_type("Str", expr_type, Str_fields, 1);
     if (!Str_type) return 0;
+    FormattedValue_type = make_type("FormattedValue", expr_type,
+                                    FormattedValue_fields, 3);
+    if (!FormattedValue_type) return 0;
+    JoinedStr_type = make_type("JoinedStr", expr_type, JoinedStr_fields, 1);
+    if (!JoinedStr_type) return 0;
     Bytes_type = make_type("Bytes", expr_type, Bytes_fields, 1);
     if (!Bytes_type) return 0;
     NameConstant_type = make_type("NameConstant", expr_type,
@@ -2114,6 +2131,42 @@ Str(string s, int lineno, int col_offset, PyArena *arena)
     return p;
 }
 
+expr_ty
+FormattedValue(expr_ty value, int conversion, expr_ty format_spec, int lineno,
+               int col_offset, PyArena *arena)
+{
+    expr_ty p;
+    if (!value) {
+        PyErr_SetString(PyExc_ValueError,
+                        "field value is required for FormattedValue");
+        return NULL;
+    }
+    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
+    if (!p)
+        return NULL;
+    p->kind = FormattedValue_kind;
+    p->v.FormattedValue.value = value;
+    p->v.FormattedValue.conversion = conversion;
+    p->v.FormattedValue.format_spec = format_spec;
+    p->lineno = lineno;
+    p->col_offset = col_offset;
+    return p;
+}
+
+expr_ty
+JoinedStr(asdl_seq * values, int lineno, int col_offset, PyArena *arena)
+{
+    expr_ty p;
+    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
+    if (!p)
+        return NULL;
+    p->kind = JoinedStr_kind;
+    p->v.JoinedStr.values = values;
+    p->lineno = lineno;
+    p->col_offset = col_offset;
+    return p;
+}
+
 expr_ty
 Bytes(bytes s, int lineno, int col_offset, PyArena *arena)
 {
@@ -3277,6 +3330,34 @@ ast2obj_expr(void* _o)
             goto failed;
         Py_DECREF(value);
         break;
+    case FormattedValue_kind:
+        result = PyType_GenericNew(FormattedValue_type, NULL, NULL);
+        if (!result) goto failed;
+        value = ast2obj_expr(o->v.FormattedValue.value);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
+            goto failed;
+        Py_DECREF(value);
+        value = ast2obj_int(o->v.FormattedValue.conversion);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_conversion, value) == -1)
+            goto failed;
+        Py_DECREF(value);
+        value = ast2obj_expr(o->v.FormattedValue.format_spec);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_format_spec, value) == -1)
+            goto failed;
+        Py_DECREF(value);
+        break;
+    case JoinedStr_kind:
+        result = PyType_GenericNew(JoinedStr_type, NULL, NULL);
+        if (!result) goto failed;
+        value = ast2obj_list(o->v.JoinedStr.values, ast2obj_expr);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_values, value) == -1)
+            goto failed;
+        Py_DECREF(value);
+        break;
     case Bytes_kind:
         result = PyType_GenericNew(Bytes_type, NULL, NULL);
         if (!result) goto failed;
@@ -6305,6 +6386,86 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
+    isinstance = PyObject_IsInstance(obj, (PyObject*)FormattedValue_type);
+    if (isinstance == -1) {
+        return 1;
+    }
+    if (isinstance) {
+        expr_ty value;
+        int conversion;
+        expr_ty format_spec;
+
+        if (_PyObject_HasAttrId(obj, &PyId_value)) {
+            int res;
+            tmp = _PyObject_GetAttrId(obj, &PyId_value);
+            if (tmp == NULL) goto failed;
+            res = obj2ast_expr(tmp, &value, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        } else {
+            PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from FormattedValue");
+            return 1;
+        }
+        if (exists_not_none(obj, &PyId_conversion)) {
+            int res;
+            tmp = _PyObject_GetAttrId(obj, &PyId_conversion);
+            if (tmp == NULL) goto failed;
+            res = obj2ast_int(tmp, &conversion, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        } else {
+            conversion = 0;
+        }
+        if (exists_not_none(obj, &PyId_format_spec)) {
+            int res;
+            tmp = _PyObject_GetAttrId(obj, &PyId_format_spec);
+            if (tmp == NULL) goto failed;
+            res = obj2ast_expr(tmp, &format_spec, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        } else {
+            format_spec = NULL;
+        }
+        *out = FormattedValue(value, conversion, format_spec, lineno,
+                              col_offset, arena);
+        if (*out == NULL) goto failed;
+        return 0;
+    }
+    isinstance = PyObject_IsInstance(obj, (PyObject*)JoinedStr_type);
+    if (isinstance == -1) {
+        return 1;
+    }
+    if (isinstance) {
+        asdl_seq* values;
+
+        if (_PyObject_HasAttrId(obj, &PyId_values)) {
+            int res;
+            Py_ssize_t len;
+            Py_ssize_t i;
+            tmp = _PyObject_GetAttrId(obj, &PyId_values);
+            if (tmp == NULL) goto failed;
+            if (!PyList_Check(tmp)) {
+                PyErr_Format(PyExc_TypeError, "JoinedStr field \"values\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                goto failed;
+            }
+            len = PyList_GET_SIZE(tmp);
+            values = _Py_asdl_seq_new(len, arena);
+            if (values == NULL) goto failed;
+            for (i = 0; i < len; i++) {
+                expr_ty value;
+                res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena);
+                if (res != 0) goto failed;
+                asdl_seq_SET(values, i, value);
+            }
+            Py_CLEAR(tmp);
+        } else {
+            PyErr_SetString(PyExc_TypeError, "required field \"values\" missing from JoinedStr");
+            return 1;
+        }
+        *out = JoinedStr(values, lineno, col_offset, arena);
+        if (*out == NULL) goto failed;
+        return 0;
+    }
     isinstance = PyObject_IsInstance(obj, (PyObject*)Bytes_type);
     if (isinstance == -1) {
         return 1;
@@ -7673,6 +7834,10 @@ PyInit__ast35(void)
     if (PyDict_SetItemString(d, "Call", (PyObject*)Call_type) < 0) return NULL;
     if (PyDict_SetItemString(d, "Num", (PyObject*)Num_type) < 0) return NULL;
     if (PyDict_SetItemString(d, "Str", (PyObject*)Str_type) < 0) return NULL;
+    if (PyDict_SetItemString(d, "FormattedValue",
+        (PyObject*)FormattedValue_type) < 0) return NULL;
+    if (PyDict_SetItemString(d, "JoinedStr", (PyObject*)JoinedStr_type) < 0)
+        return NULL;
     if (PyDict_SetItemString(d, "Bytes", (PyObject*)Bytes_type) < 0) return
         NULL;
     if (PyDict_SetItemString(d, "NameConstant", (PyObject*)NameConstant_type) <