Add experiment to produce precompiled builds of serde_derive

precompiled/proc-macro2/src/watt/buffer.rs

@@ -0,0 +1,69 @@
+use std::str;
+
+pub struct OutputBuffer {
+    bytes: Vec<u8>,
+}
+
+impl OutputBuffer {
+    pub fn new() -> Self {
+        OutputBuffer { bytes: Vec::new() }
+    }
+
+    pub fn write_u8(&mut self, value: u8) {
+        self.bytes.push(value);
+    }
+
+    pub fn write_u16(&mut self, value: u16) {
+        self.bytes.extend_from_slice(&value.to_le_bytes());
+    }
+
+    pub fn write_u32(&mut self, value: u32) {
+        self.bytes.extend_from_slice(&value.to_le_bytes());
+    }
+
+    pub fn write_str(&mut self, value: &str) {
+        self.bytes.extend_from_slice(value.as_bytes());
+    }
+
+    pub fn into_bytes(self) -> Vec<u8> {
+        self.bytes
+    }
+}
+
+pub struct InputBuffer<'a> {
+    bytes: &'a [u8],
+}
+
+impl<'a> InputBuffer<'a> {
+    pub fn new(bytes: &'a [u8]) -> Self {
+        InputBuffer { bytes }
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.bytes.is_empty()
+    }
+
+    pub fn read_u8(&mut self) -> u8 {
+        let (first, rest) = self.bytes.split_first().unwrap();
+        self.bytes = rest;
+        *first
+    }
+
+    pub fn read_u16(&mut self) -> u16 {
+        let (value, rest) = self.bytes.split_at(2);
+        self.bytes = rest;
+        u16::from_le_bytes([value[0], value[1]])
+    }
+
+    pub fn read_u32(&mut self) -> u32 {
+        let (value, rest) = self.bytes.split_at(4);
+        self.bytes = rest;
+        u32::from_le_bytes([value[0], value[1], value[2], value[3]])
+    }
+
+    pub fn read_str(&mut self, len: usize) -> &'a str {
+        let (string, rest) = self.bytes.split_at(len);
+        self.bytes = rest;
+        str::from_utf8(string).unwrap()
+    }
+}

precompiled/proc-macro2/src/watt/bytecode.rs

@@ -0,0 +1,17 @@
+pub enum Bytecode {}
+
+impl Bytecode {
+    pub const GROUP_PARENTHESIS: u8 = 0;
+    pub const GROUP_BRACE: u8 = 1;
+    pub const GROUP_BRACKET: u8 = 2;
+    pub const GROUP_NONE: u8 = 3;
+    pub const IDENT: u8 = 4;
+    pub const PUNCT_ALONE: u8 = 5;
+    pub const PUNCT_JOINT: u8 = 6;
+    pub const LITERAL: u8 = 7;
+    pub const LOAD_GROUP: u8 = 8;
+    pub const LOAD_IDENT: u8 = 9;
+    pub const LOAD_PUNCT: u8 = 10;
+    pub const LOAD_LITERAL: u8 = 11;
+    pub const SET_SPAN: u8 = 12;
+}

precompiled/proc-macro2/src/watt/mod.rs

@@ -0,0 +1,196 @@
+pub mod buffer;
+pub mod bytecode;
+
+use crate::watt::buffer::{InputBuffer, OutputBuffer};
+use crate::watt::bytecode::Bytecode;
+use crate::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree};
+use std::str::FromStr;
+
+pub enum Kind {
+    Group(Delimiter),
+    Ident,
+    Punct(Spacing),
+    Literal,
+}
+
+pub enum Identity {}
+
+impl Identity {
+    pub const RESPANNED: u32 = 1 << 31;
+    pub const NOVEL: u32 = u32::MAX;
+}
+
+impl Span {
+    fn is_call_site(&self) -> bool {
+        self.lo == 0 && self.hi == 0
+    }
+}
+
+fn post_increment(counter: &mut u32) -> impl FnMut() -> u32 + '_ {
+    || {
+        let value = *counter;
+        *counter += 1;
+        value
+    }
+}
+
+pub fn load(buf: &mut InputBuffer) -> TokenStream {
+    let mut span_counter = 0;
+    let mut next_span = post_increment(&mut span_counter);
+    let mut next_span = || {
+        let next = next_span();
+        Span { lo: next, hi: next }
+    };
+
+    let [mut group_counter, mut ident_counter, mut punct_counter, mut literal_counter] = [0; 4];
+    let mut next_group = post_increment(&mut group_counter);
+    let mut next_ident = post_increment(&mut ident_counter);
+    let mut next_punct = post_increment(&mut punct_counter);
+    let mut next_literal = post_increment(&mut literal_counter);
+
+    let mut trees = Vec::new();
+    while !buf.is_empty() {
+        match match buf.read_u8() {
+            Bytecode::GROUP_PARENTHESIS => Kind::Group(Delimiter::Parenthesis),
+            Bytecode::GROUP_BRACE => Kind::Group(Delimiter::Brace),
+            Bytecode::GROUP_BRACKET => Kind::Group(Delimiter::Bracket),
+            Bytecode::GROUP_NONE => Kind::Group(Delimiter::None),
+            Bytecode::IDENT => Kind::Ident,
+            Bytecode::PUNCT_ALONE => Kind::Punct(Spacing::Alone),
+            Bytecode::PUNCT_JOINT => Kind::Punct(Spacing::Joint),
+            Bytecode::LITERAL => Kind::Literal,
+            _ => unreachable!(),
+        } {
+            Kind::Group(delimiter) => {
+                let len = buf.read_u32();
+                let stream = trees.drain(trees.len() - len as usize..).collect();
+                trees.push(TokenTree::Group(Group {
+                    delimiter,
+                    stream,
+                    span: next_span(),
+                    span_open: next_span(),
+                    span_close: next_span(),
+                    identity: next_group(),
+                }));
+            }
+            Kind::Ident => {
+                let len = buf.read_u16();
+                let repr = buf.read_str(len as usize);
+                trees.push(TokenTree::Ident(Ident {
+                    fallback: proc_macro2::Ident::new(repr, proc_macro2::Span::call_site()),
+                    span: next_span(),
+                    identity: next_ident(),
+                }));
+            }
+            Kind::Punct(spacing) => {
+                let ch = buf.read_u8();
+                assert!(ch.is_ascii());
+                trees.push(TokenTree::Punct(Punct {
+                    fallback: proc_macro2::Punct::new(ch as char, spacing),
+                    span: next_span(),
+                    identity: next_punct(),
+                }));
+            }
+            Kind::Literal => {
+                let len = buf.read_u16();
+                let repr = buf.read_str(len as usize);
+                trees.push(TokenTree::Literal(Literal {
+                    fallback: proc_macro2::Literal::from_str(repr).unwrap(),
+                    span: next_span(),
+                    identity: next_literal(),
+                }));
+            }
+        }
+    }
+
+    TokenStream { content: trees }
+}
+
+pub fn linearize(tokens: TokenStream) -> Vec<u8> {
+    let mut buf = OutputBuffer::new();
+    for token in &tokens.content {
+        linearize_token(token, &mut buf);
+    }
+    buf.into_bytes()
+}
+
+fn linearize_token(token: &TokenTree, buf: &mut OutputBuffer) {
+    let needs_span;
+    match token {
+        TokenTree::Group(group) => {
+            if group.identity < Identity::NOVEL {
+                buf.write_u8(Bytecode::LOAD_GROUP);
+                buf.write_u32(group.identity & !Identity::RESPANNED);
+                needs_span = group.identity >= Identity::RESPANNED;
+            } else {
+                let len = group.stream.len();
+                assert!(len <= u32::MAX as usize);
+                for token in &group.stream {
+                    linearize_token(token, buf);
+                }
+                buf.write_u8(match group.delimiter {
+                    Delimiter::Parenthesis => Bytecode::GROUP_PARENTHESIS,
+                    Delimiter::Brace => Bytecode::GROUP_BRACE,
+                    Delimiter::Bracket => Bytecode::GROUP_BRACKET,
+                    Delimiter::None => Bytecode::GROUP_NONE,
+                });
+                buf.write_u32(len as u32);
+                needs_span = !group.span.is_call_site();
+            }
+        }
+        TokenTree::Ident(ident) => {
+            if ident.identity < Identity::NOVEL {
+                buf.write_u8(Bytecode::LOAD_IDENT);
+                buf.write_u32(ident.identity & !Identity::RESPANNED);
+                needs_span = ident.identity >= Identity::RESPANNED;
+            } else {
+                buf.write_u8(Bytecode::IDENT);
+                let repr = ident.to_string();
+                assert!(repr.len() <= u16::MAX as usize);
+                buf.write_u16(repr.len() as u16);
+                buf.write_str(&repr);
+                linearize_span(ident.span, buf);
+                needs_span = false;
+            }
+        }
+        TokenTree::Punct(punct) => {
+            if punct.identity < Identity::NOVEL {
+                buf.write_u8(Bytecode::LOAD_PUNCT);
+                buf.write_u32(punct.identity & !Identity::RESPANNED);
+                needs_span = punct.identity >= Identity::RESPANNED;
+            } else {
+                buf.write_u8(match punct.spacing() {
+                    Spacing::Alone => Bytecode::PUNCT_ALONE,
+                    Spacing::Joint => Bytecode::PUNCT_JOINT,
+                });
+                let ch = punct.as_char();
+                assert!(ch.is_ascii());
+                buf.write_u8(ch as u8);
+                needs_span = !punct.span.is_call_site();
+            }
+        }
+        TokenTree::Literal(literal) => {
+            if literal.identity < Identity::NOVEL {
+                buf.write_u8(Bytecode::LOAD_LITERAL);
+                buf.write_u32(literal.identity & !Identity::RESPANNED);
+                needs_span = literal.identity >= Identity::RESPANNED;
+            } else {
+                buf.write_u8(Bytecode::LITERAL);
+                let repr = literal.to_string();
+                assert!(repr.len() <= u16::MAX as usize);
+                buf.write_u16(repr.len() as u16);
+                buf.write_str(&repr);
+                needs_span = !literal.span.is_call_site();
+            }
+        }
+    }
+    if needs_span {
+        buf.write_u8(Bytecode::SET_SPAN);
+        linearize_span(token.span(), buf);
+    }
+}
+
+fn linearize_span(span: Span, buf: &mut OutputBuffer) {
+    buf.write_u32(span.lo);
+    buf.write_u32(span.hi);
+}