google · hchataing · Aug 31, 2023 · Aug 27, 2023 · Aug 27, 2023 · Aug 31, 2023
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,4 +1,5 @@
 [workspace]
 members = [
     "pdl-compiler",
+    "pdl-derive",
 ]
diff --git a/doc/rust-generated-code-guide.rst b/doc/rust-generated-code-guide.rst
@@ -4,7 +4,23 @@ Rust Generated Code Guide
 Usage
 -----
 
-Example invocation:
+The crate `pdl_derive` lets developers embed their grammar in a source module
+using the `pdl` proc_macro attribute. Example usage:
+
+.. sourcecode:: rust
+
+        use pdl_derive::pdl
+
+        #[pdl("my-protocol.pdl")]
+        mod my_protocol {
+        }
+
+The `pdl` proc_macro attribute must be attached to a module declaration.
+`pdl` preserves the original name, attributes, and items of the associated
+module.
+
+The backend can also be pre-generated from the `pdlc` tool,
+and compiled as source. Example invocation:
 
 .. sourcecode:: bash
 

diff --git a/pdl-compiler/examples/jpeg.pdl → examples/jpeg.pdl b/pdl-compiler/examples/jpeg.pdl → examples/jpeg.pdl
@@ -61,13 +61,13 @@ enum MarkerType : 8 {
 
 struct Marker {
     _fixed_ = 0xff : 8,
-    type: MarkerType,
+    type : MarkerType,
     _payload_,
 }
 
 struct Segment: Marker {
-    _size_(_payload_): 16,
-    _payload_ [+2],
+    _size_(_payload_) : 16,
+    _payload_ : [+2],
 }
 
 struct StartOfImage : Marker(type = SOI) {}

diff --git a/pdl-compiler/examples/pcap.pdl → examples/pcap.pdl b/pdl-compiler/examples/pcap.pdl → examples/pcap.pdl
diff --git a/pdl-compiler/src/analyzer.rs b/pdl-compiler/src/analyzer.rs
@@ -1681,8 +1681,7 @@ mod test {
     macro_rules! raises {
         ($code:ident, $text:literal) => {{
             let mut db = SourceDatabase::new();
-            let file = parse_inline(&mut db, "stdin".to_owned(), $text.to_owned())
-                .expect("parsing failure");
+            let file = parse_inline(&mut db, "stdin", $text.to_owned()).expect("parsing failure");
             let result = analyzer::analyze(&file);
             assert!(matches!(result, Err(_)));
             let diagnostics = result.err().unwrap();
@@ -1697,8 +1696,7 @@ mod test {
     macro_rules! valid {
         ($text:literal) => {{
             let mut db = SourceDatabase::new();
-            let file = parse_inline(&mut db, "stdin".to_owned(), $text.to_owned())
-                .expect("parsing failure");
+            let file = parse_inline(&mut db, "stdin", $text.to_owned()).expect("parsing failure");
             assert!(analyzer::analyze(&file).is_ok());
         }};
     }
@@ -2750,8 +2748,7 @@ mod test {
 
     fn annotations(text: &str) -> Vec<Annotations> {
         let mut db = SourceDatabase::new();
-        let file =
-            parse_inline(&mut db, "stdin".to_owned(), text.to_owned()).expect("parsing failure");
+        let file = parse_inline(&mut db, "stdin", text.to_owned()).expect("parsing failure");
         let file = analyzer::analyze(&file).expect("analyzer failure");
         file.declarations
             .iter()
@@ -3255,8 +3252,7 @@ mod test {
 
     fn desugar(text: &str) -> analyzer::ast::File {
         let mut db = SourceDatabase::new();
-        let file =
-            parse_inline(&mut db, "stdin".to_owned(), text.to_owned()).expect("parsing failure");
+        let file = parse_inline(&mut db, "stdin", text.to_owned()).expect("parsing failure");
         analyzer::analyze(&file).expect("analyzer failure")
     }
 

diff --git a/pdl-compiler/src/backends/rust.rs b/pdl-compiler/src/backends/rust.rs
@@ -985,18 +985,28 @@ fn generate_decl(
 ///
 /// The code is not formatted, pipe it through `rustfmt` to get
 /// readable source code.
-pub fn generate(sources: &ast::SourceDatabase, file: &analyzer_ast::File) -> String {
+pub fn generate_tokens(
+    sources: &ast::SourceDatabase,
+    file: &analyzer_ast::File,
+) -> proc_macro2::TokenStream {
     let source = sources.get(file.file).expect("could not read source");
     let preamble = preamble::generate(Path::new(source.name()));
 
     let scope = analyzer::Scope::new(file).expect("could not create scope");
     let decls = file.declarations.iter().map(|decl| generate_decl(&scope, file, decl));
-    let code = quote! {
+    quote! {
         #preamble
 
         #(#decls)*
-    };
-    let syntax_tree = syn::parse2(code).expect("Could not parse code");
+    }
+}
+
+/// Generate formatted Rust code from an AST.
+///
+/// The code is not formatted, pipe it through `rustfmt` to get
+/// readable source code.
+pub fn generate(sources: &ast::SourceDatabase, file: &analyzer_ast::File) -> String {
+    let syntax_tree = syn::parse2(generate_tokens(sources, file)).expect("Could not parse code");
     prettyplease::unparse(&syntax_tree)
 }
 
@@ -1017,8 +1027,7 @@ mod tests {
     /// Panics on parse errors.
     pub fn parse_str(text: &str) -> analyzer_ast::File {
         let mut db = ast::SourceDatabase::new();
-        let file =
-            parse_inline(&mut db, String::from("stdin"), String::from(text)).expect("parse error");
+        let file = parse_inline(&mut db, "stdin", String::from(text)).expect("parse error");
         analyzer::analyze(&file).expect("analyzer error")
     }
 
@@ -1083,7 +1092,7 @@ mod tests {
                     let endianness = stringify!($endianness);
                     let code = format!("{endianness}_packets\n{}", $code);
                     let mut db = ast::SourceDatabase::new();
-                    let file = parse_inline(&mut db, String::from("test"), code).unwrap();
+                    let file = parse_inline(&mut db, "test", code).unwrap();
                     let file = analyzer::analyze(&file).unwrap();
                     let actual_code = generate(&db, &file);
                     assert_snapshot_eq(

diff --git a/pdl-compiler/src/backends/rust/parser.rs b/pdl-compiler/src/backends/rust/parser.rs
@@ -741,8 +741,7 @@ mod tests {
     /// Panics on parse errors.
     pub fn parse_str(text: &str) -> analyzer_ast::File {
         let mut db = ast::SourceDatabase::new();
-        let file =
-            parse_inline(&mut db, String::from("stdin"), String::from(text)).expect("parse error");
+        let file = parse_inline(&mut db, "stdin", String::from(text)).expect("parse error");
         analyzer::analyze(&file).expect("analyzer error")
     }
 

diff --git a/pdl-compiler/src/backends/rust_no_allocation/test.rs b/pdl-compiler/src/backends/rust_no_allocation/test.rs
@@ -215,7 +215,7 @@ pub fn generate_test_file() -> Result<String, String> {
         serde_json::from_str(file).map_err(|_| "could not parse test vectors")?;
 
     let pdl = include_str!("../../../tests/canonical/le_rust_noalloc_test_file.pdl");
-    let ast = parse_inline(&mut ast::SourceDatabase::new(), "test.pdl".to_owned(), pdl.to_owned())
+    let ast = parse_inline(&mut ast::SourceDatabase::new(), "test.pdl", pdl.to_owned())
         .expect("could not parse reference PDL");
     let packet_lookup =
         ast.declarations

diff --git a/pdl-compiler/src/main.rs b/pdl-compiler/src/main.rs
@@ -89,7 +89,7 @@ fn main() -> Result<(), String> {
     }
 
     let mut sources = ast::SourceDatabase::new();
-    match parser::parse_file(&mut sources, opt.input_file) {
+    match parser::parse_file(&mut sources, &opt.input_file) {
         Ok(file) => {
             let file = filter_declarations(file, &opt.exclude_declaration);
             let analyzed_file = match analyzer::analyze(&file) {

diff --git a/pdl-compiler/src/parser.rs b/pdl-compiler/src/parser.rs
@@ -631,18 +631,18 @@ fn parse_toplevel(root: Node<'_>, context: &Context) -> Result<ast::File, String
 /// name.
 pub fn parse_inline(
     sources: &mut crate::ast::SourceDatabase,
-    name: String,
+    name: &str,
     source: String,
 ) -> Result<ast::File, Diagnostic<crate::ast::FileId>> {
     let root = PDLParser::parse(Rule::file, &source)
         .map_err(|e| {
             Diagnostic::error()
-                .with_message(format!("failed to parse input file '{}': {}", &name, e))
+                .with_message(format!("failed to parse input file '{}': {}", name, e))
         })?
         .next()
         .unwrap();
     let line_starts: Vec<_> = files::line_starts(&source).collect();
-    let file = sources.add(name, source.clone());
+    let file = sources.add(name.to_owned(), source.clone());
     parse_toplevel(root, &(file, &line_starts)).map_err(|e| Diagnostic::error().with_message(e))
 }
 
@@ -653,10 +653,10 @@ pub fn parse_inline(
 /// message in case of syntax error.
 pub fn parse_file(
     sources: &mut crate::ast::SourceDatabase,
-    name: String,
+    name: &str,
 ) -> Result<ast::File, Diagnostic<crate::ast::FileId>> {
-    let source = std::fs::read_to_string(&name).map_err(|e| {
-        Diagnostic::error().with_message(format!("failed to read input file '{}': {}", &name, e))
+    let source = std::fs::read_to_string(name).map_err(|e| {
+        Diagnostic::error().with_message(format!("failed to read input file '{}': {}", name, e))
     })?;
     parse_inline(sources, name, source)
 }
@@ -670,9 +670,7 @@ mod test {
         // The file starts out with a placeholder little-endian value.
         // This tests that we update it while parsing.
         let mut db = crate::ast::SourceDatabase::new();
-        let file =
-            parse_inline(&mut db, String::from("stdin"), String::from("  big_endian_packets  "))
-                .unwrap();
+        let file = parse_inline(&mut db, "stdin", String::from("  big_endian_packets  ")).unwrap();
         assert_eq!(file.endianness.value, crate::ast::EndiannessValue::BigEndian);
         assert_ne!(file.endianness.loc, crate::ast::SourceRange::default());
     }
@@ -709,7 +707,7 @@ mod test {
         let mut db = crate::ast::SourceDatabase::new();
         assert!(parse_inline(
             &mut db,
-            "test".to_owned(),
+            "test",
             r#"
             little_endian_packetsstructx{foo:8}
             "#
@@ -719,7 +717,7 @@ mod test {
 
         let result = parse_inline(
             &mut db,
-            "test".to_owned(),
+            "test",
             r#"
             little_endian_packets
             struct x { foo:8 }

diff --git a/pdl-derive/Cargo.toml b/pdl-derive/Cargo.toml
@@ -0,0 +1,32 @@
+[package]
+name = "pdl-derive"
+version = "0.1.6"
+edition = "2021"
+description = "Parser and serializer generator for protocol binary packets"
+repository = "https://github.com/google/pdl/"
+license = "Apache-2.0"
+readme = "../README.md"
+keywords = ["pdl", "parser", "serializer", "grammar"]
+authors = [
+    "Henri Chataing <henrichataing@google.com>",
+    "David de Jesus Duarte <licorne@google.com>",
+    "Martin Geisler <mgeisler@google.com>"
+]
+exclude = ["editors/*"]
+categories = ["parsing"]
+
+[lib]
+name = "pdl_derive"
+proc-macro = true
+
+[dependencies]
+codespan-reporting = "0.11.1"
+pdl-compiler = {path = "../pdl-compiler", version = "0.1.6"}
+proc-macro2 = "1.0.66"
+quote = "1.0.33"
+syn = {version = "2.0.29", features = ["full"]}
+termcolor = "1.2.0"
+
+[dev-dependencies]
+bytes = "1.4.0"
+thiserror = "1.0.47"