[rust-compiler] add the extract-graphql crate

This create is a simple, but extremely fast way to extract Relay
`graphql` tags from source files. It's a basic tokenizer for "js like"
languages including Typescript and Flow.

compiler/crates/extract-graphql/Cargo.toml

@@ -0,0 +1,7 @@
+[project]
+name = "extract-graphql"
+version = "0.0.0"
+edition = "2018"
+
+[dev-dependencies]
+fixture-tests = { path = "../fixture-tests" }

compiler/crates/extract-graphql/src/lib.rs

@@ -0,0 +1,139 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#![deny(warnings)]
+#![deny(rust_2018_idioms)]
+#![deny(clippy::all)]
+
+use std::iter::Peekable;
+use std::str::CharIndices;
+
+type IndexedCharIter<'a> = Peekable<CharIndices<'a>>;
+
+/// Extract graphql`text` literals from JS-like code. This should work for Flow
+/// or TypeScript alike.
+pub fn parse_chunks(input: &str) -> Result<Vec<&str>, String> {
+    if !input.contains("graphql`") {
+        return Ok(vec![]);
+    }
+    let mut res = vec![];
+    let mut it = input.char_indices().peekable();
+    'code: while let Some((i, c)) = it.next() {
+        match c {
+            'g' => {
+                for expected in ['r', 'a', 'p', 'h', 'q', 'l', '`'].iter() {
+                    if let Some((_, c)) = it.next() {
+                        if c != *expected {
+                            consume_identifier(&mut it);
+                            continue 'code;
+                        }
+                    }
+                }
+                let start = i;
+                while let Some((i, c)) = it.next() {
+                    match c {
+                        '`' => {
+                            let end = i;
+                            res.push(&input[start + 8..end]);
+                            continue 'code;
+                        }
+                        '$' => {
+                            if let Some((_, '{')) = it.next() {
+                                return Err("graphql literals cannot have string substitutions."
+                                    .to_string());
+                            }
+                        }
+                        _ => {}
+                    }
+                }
+            }
+            'a'..='z' | 'A'..='Z' | '_' => {
+                consume_identifier(&mut it);
+            }
+            // Skip over template literals. Unfortunately, this isn't enough to
+            // deal with nested template literals and runs a risk of skipping
+            // over too much code.
+            //   '`' => {
+            //       while let Some((_, c)) = it.next() {
+            //           match c {
+            //               '`' => {
+            //                   continue 'code;
+            //               }
+            //               '\\' => {
+            //                   it.next();
+            //               }
+            //               _ => {}
+            //           }
+            //       }
+            //   }
+            '"' => consume_string(&mut it, '"'),
+            '\'' => consume_string(&mut it, '\''),
+            '/' => match it.next() {
+                Some((_, '/')) => {
+                    consume_line_comment(&mut it);
+                }
+                Some((_, '*')) => {
+                    consume_block_comment(&mut it);
+                }
+                _ => {}
+            },
+            _ => {}
+        };
+    }
+    Ok(res)
+}
+
+fn consume_identifier(it: &mut IndexedCharIter<'_>) {
+    for (_, c) in it {
+        match c {
+            'a'..='z' | 'A'..='Z' | '_' | '0'..='9' => {}
+            _ => {
+                return;
+            }
+        }
+    }
+}
+
+fn consume_line_comment(it: &mut IndexedCharIter<'_>) {
+    for (_, c) in it {
+        match c {
+            '\n' | '\r' => {
+                break;
+            }
+            _ => {}
+        }
+    }
+}
+
+fn consume_block_comment(it: &mut IndexedCharIter<'_>) {
+    while let Some((_, c)) = it.next() {
+        if c == '*' {
+            if let Some((_, '/')) = it.peek() {
+                it.next();
+                break;
+            }
+        }
+    }
+}
+
+fn consume_string(it: &mut IndexedCharIter<'_>, quote: char) {
+    while let Some((_, c)) = it.next() {
+        match c {
+            '\\' => {
+                it.next();
+            }
+            '\'' | '"' if c == quote => {
+                return;
+            }
+            '\n' | '\r' => {
+                // Unexpected newline, terminate the string parsing to recover
+                return;
+            }
+            _ => {}
+        }
+    }
+}

compiler/crates/extract-graphql/tests/extract/fixtures/comments.expected

@@ -0,0 +1,25 @@
+==================================== INPUT ====================================
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// graphql`ignored`
+
+/*
+graphql`ignored`
+*/
+
+/* ... */ graphql`not ignored 1`
+
+/**
+ * block
+ **/
+graphql`not ignored 2`
+==================================== OUTPUT ===================================
+[
+    "not ignored 1",
+    "not ignored 2",
+]

compiler/crates/extract-graphql/tests/extract/fixtures/comments.js

@@ -0,0 +1,19 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// graphql`ignored`
+
+/*
+graphql`ignored`
+*/
+
+/* ... */ graphql`not ignored 1`
+
+/**
+ * block
+ **/
+graphql`not ignored 2`

compiler/crates/extract-graphql/tests/extract/fixtures/inline.invalid.expected

@@ -0,0 +1,19 @@
+==================================== INPUT ====================================
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// expected-to-throw
+
+blah
+
+graphql`
+    fragment Foo on ${type} {
+        __typename
+    }
+`
+==================================== ERROR ====================================
+graphql literals cannot have string substitutions.

compiler/crates/extract-graphql/tests/extract/fixtures/inline.invalid.js

@@ -0,0 +1,16 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// expected-to-throw
+
+blah
+
+graphql`
+    fragment Foo on ${type} {
+        __typename
+    }
+`

compiler/crates/extract-graphql/tests/extract/fixtures/nested_template_literals.expected

@@ -0,0 +1,20 @@
+==================================== INPUT ====================================
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+const string = `
+  ${line1 != null ? `${line1}\n` : ''}
+  ${line2 != null ? `${line2}\n` : ''}
+  ${city != null && state != null ? `${city}, ${state}\n` : ''}
+  ${postalCode != null ? `${postalCode}\n` : ''}
+`;
+
+graphql`some graphql`;
+==================================== OUTPUT ===================================
+[
+    "some graphql",
+]

compiler/crates/extract-graphql/tests/extract/fixtures/nested_template_literals.js

@@ -0,0 +1,15 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+const string = `
+  ${line1 != null ? `${line1}\n` : ''}
+  ${line2 != null ? `${line2}\n` : ''}
+  ${city != null && state != null ? `${city}, ${state}\n` : ''}
+  ${postalCode != null ? `${postalCode}\n` : ''}
+`;
+
+graphql`some graphql`;

compiler/crates/extract-graphql/tests/extract/fixtures/no_graphql.expected

@@ -0,0 +1,22 @@
+==================================== INPUT ====================================
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+const oops = [
+    graphqlgraphql`not matching1`,
+    graphqlabc`not matching2`,
+    _graphql`not matching3`,
+    abcgraphql`not matching4`,
+    " graphql`in string` ",
+    " \" graphql`in string` ",
+    ' graphql`in string` ',
+    ' \' graphql`in string` ',
+    // graphql`in comment`
+    /* graphql`in comment` */
+];
+==================================== OUTPUT ===================================
+[]

compiler/crates/extract-graphql/tests/extract/fixtures/no_graphql.js

@@ -0,0 +1,19 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+const oops = [
+    graphqlgraphql`not matching1`,
+    graphqlabc`not matching2`,
+    _graphql`not matching3`,
+    abcgraphql`not matching4`,
+    " graphql`in string` ",
+    " \" graphql`in string` ",
+    ' graphql`in string` ',
+    ' \' graphql`in string` ',
+    // graphql`in comment`
+    /* graphql`in comment` */
+];

compiler/crates/extract-graphql/tests/extract/fixtures/quote_in_jsx.expected

@@ -0,0 +1,14 @@
+==================================== INPUT ====================================
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+<Test>'</Test>
+graphql`fragment on User`
+==================================== OUTPUT ===================================
+[
+    "fragment on User",
+]

compiler/crates/extract-graphql/tests/extract/fixtures/quote_in_jsx.js

@@ -0,0 +1,9 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+<Test>'</Test>
+graphql`fragment on User`

compiler/crates/extract-graphql/tests/extract/fixtures/simple.expected

@@ -0,0 +1,13 @@
+==================================== INPUT ====================================
+function MyComponent() {
+  useFragment(graphql`
+    fragment Test on User {
+      __typename
+    }
+  `, user)
+  return <div>Test</div>;
+}
+==================================== OUTPUT ===================================
+[
+    "\n    fragment Test on User {\n      __typename\n    }\n  ",
+]

compiler/crates/extract-graphql/tests/extract/fixtures/simple.flow

@@ -0,0 +1,8 @@
+function MyComponent() {
+  useFragment(graphql`
+    fragment Test on User {
+      __typename
+    }
+  `, user)
+  return <div>Test</div>;
+}

compiler/crates/extract-graphql/tests/extract/mod.rs

@@ -0,0 +1,13 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+use extract_graphql::parse_chunks;
+use fixture_tests::Fixture;
+
+pub fn transform_fixture(fixture: &Fixture) -> Result<String, String> {
+    parse_chunks(fixture.content).map(|chunks| format!("{:#?}", chunks))
+}