Merge pull request #204 from JohnHeitmann/master

Implemented disallow_unknown

README.md

@@ -585,6 +585,8 @@ Annotations
 `serde_codegen` and `serde_macros` support annotations that help to customize
 how types are serialized. Here are the supported annotations:
 
+Field Annotations:
+
 | Annotation                                   | Function                                                       |
 | ----------                                   | --------                                                       |
 | `#[serde(rename(json="name1", xml="name2"))` | Serialize this field with the given name for the given formats |
@@ -594,6 +596,13 @@ how types are serialized. Here are the supported annotations:
 | `#[serde(skip_serializing_if_empty)`         | Do not serialize this value if `$value.is_empty()` is `true`   |
 | `#[serde(skip_serializing_if_none)`          | Do not serialize this value if `$value.is_none()` is `true`    |
 
+Structure Annotations:
+
+| Annotation                  | Function                                                                                                                                           |
+| ----------                  | --------                                                                                                                                           |
+| `#[serde(deny_unknown_fields)` | Always error during serialization when encountering unknown fields. When absent, unknown fields are ignored for self-describing formats like JSON. |
+
+
 Serialization Formats Using Serde
 =================================
 

serde/src/de/impls.rs

@@ -896,6 +896,110 @@ impl<T, E> Deserialize for Result<T, E> where T: Deserialize, E: Deserialize {
     }
 }
 
+///////////////////////////////////////////////////////////////////////////////
+
+/// A target for deserializers that want to ignore data. Implements
+/// Deserialize and silently eats data given to it.
+pub struct IgnoredAny;
+
+impl Deserialize for IgnoredAny {
+    #[inline]
+    fn deserialize<D>(deserializer: &mut D) -> Result<IgnoredAny, D::Error>
+        where D: Deserializer,
+    {
+        struct IgnoredAnyVisitor;
+
+        impl Visitor for IgnoredAnyVisitor {
+            type Value = IgnoredAny;
+
+            #[inline]
+            fn visit_bool<E>(&mut self, _: bool) -> Result<IgnoredAny, E> {
+                Ok(IgnoredAny)
+            }
+
+            #[inline]
+            fn visit_i64<E>(&mut self, _: i64) -> Result<IgnoredAny, E> {
+                Ok(IgnoredAny)
+            }
+
+            #[inline]
+            fn visit_u64<E>(&mut self, _: u64) -> Result<IgnoredAny, E> {
+                Ok(IgnoredAny)
+            }
+
+            #[inline]
+            fn visit_f64<E>(&mut self, _: f64) -> Result<IgnoredAny, E> {
+                Ok(IgnoredAny)
+            }
+
+            #[inline]
+            fn visit_str<E>(&mut self, _: &str) -> Result<IgnoredAny, E>
+                where E: Error,
+            {
+                Ok(IgnoredAny)
+            }
+
+            #[inline]
+            fn visit_none<E>(&mut self) -> Result<IgnoredAny, E> {
+                Ok(IgnoredAny)
+            }
+
+            #[inline]
+            fn visit_some<D>(&mut self, _: &mut D) -> Result<IgnoredAny, D::Error>
+                where D: Deserializer,
+            {
+                Ok(IgnoredAny)
+            }
+
+            #[inline]
+            fn visit_newtype_struct<D>(&mut self, _: &mut D) -> Result<IgnoredAny, D::Error>
+                where D: Deserializer,
+            {
+                Ok(IgnoredAny)
+            }
+
+            #[inline]
+            fn visit_unit<E>(&mut self) -> Result<IgnoredAny, E> {
+                Ok(IgnoredAny)
+            }
+
+            #[inline]
+            fn visit_seq<V>(&mut self, mut visitor: V) -> Result<IgnoredAny, V::Error>
+                where V: SeqVisitor,
+            {
+                while let Some(_) = try!(visitor.visit::<IgnoredAny>()) {
+                    // Gobble
+                }
+
+                try!(visitor.end());
+                Ok(IgnoredAny)
+            }
+
+            #[inline]
+            fn visit_map<V>(&mut self, mut visitor: V) -> Result<IgnoredAny, V::Error>
+                where V: MapVisitor,
+            {
+                while let Some((_, _)) = try!(visitor.visit::<IgnoredAny, IgnoredAny>()) {
+                    // Gobble
+                }
+
+                try!(visitor.end());
+                Ok(IgnoredAny)
+            }
+
+            #[inline]
+            fn visit_bytes<E>(&mut self, _: &[u8]) -> Result<IgnoredAny, E>
+                where E: Error,
+            {
+                Ok(IgnoredAny)
+            }
+        }
+
+        deserializer.deserialize(IgnoredAnyVisitor)
+    }
+}
+
+
 ///////////////////////////////////////////////////////////////////////////////
 
 #[cfg(feature = "num-bigint")]

serde_codegen/src/attr.rs

@@ -243,3 +243,66 @@ impl<'a> FieldAttrsBuilder<'a> {
         }
     }
 }
+
+/// Represents container (e.g. struct) attribute information
+#[derive(Debug)]
+pub struct ContainerAttrs {
+    deny_unknown_fields: bool,
+}
+
+impl ContainerAttrs {
+    pub fn deny_unknown_fields(&self) -> bool {
+        self.deny_unknown_fields
+    }
+}
+
+pub struct ContainerAttrsBuilder {
+    deny_unknown_fields: bool,
+}
+
+impl ContainerAttrsBuilder {
+    pub fn new() -> ContainerAttrsBuilder {
+        ContainerAttrsBuilder {
+            deny_unknown_fields: false,
+        }
+    }
+
+    pub fn attrs(self, attrs: &[ast::Attribute]) -> ContainerAttrsBuilder {
+        attrs.iter().fold(self, ContainerAttrsBuilder::attr)
+    }
+
+    pub fn attr(self, attr: &ast::Attribute) -> ContainerAttrsBuilder {
+        match attr.node.value.node {
+            ast::MetaList(ref name, ref items) if name == &"serde" => {
+                attr::mark_used(&attr);
+                items.iter().fold(self, ContainerAttrsBuilder::meta_item)
+            }
+            _ => {
+                self
+            }
+        }
+    }
+
+    pub fn meta_item(self, meta_item: &P<ast::MetaItem>) -> ContainerAttrsBuilder {
+        match meta_item.node {
+            ast::MetaWord(ref name) if name == &"deny_unknown_fields" => {
+                self.deny_unknown_fields()
+            }
+            _ => {
+                // Ignore unknown meta variables for now.
+                self
+            }
+        }
+    }
+
+    pub fn deny_unknown_fields(mut self) -> ContainerAttrsBuilder {
+        self.deny_unknown_fields = true;
+        self
+    }
+
+    pub fn build(self) -> ContainerAttrs {
+        ContainerAttrs {
+            deny_unknown_fields: self.deny_unknown_fields,
+        }
+    }
+}

serde_codegen/src/de.rs

@@ -14,7 +14,7 @@ use syntax::ext::base::{Annotatable, ExtCtxt};
 use syntax::ext::build::AstBuilder;
 use syntax::ptr::P;
 
-use attr;
+use attr::{self, ContainerAttrs};
 use field;
 
 pub fn expand_derive_deserialize(
@@ -82,6 +82,8 @@ fn deserialize_body(
     impl_generics: &ast::Generics,
     ty: P<ast::Ty>,
 ) -> P<ast::Expr> {
+    let container_attrs = field::container_attrs(cx, item);
+
     match item.node {
         ast::ItemStruct(ref variant_data, _) => {
             deserialize_item_struct(
@@ -91,6 +93,7 @@ fn deserialize_body(
                 impl_generics,
                 ty,
                 variant_data,
+                &container_attrs,
             )
         }
         ast::ItemEnum(ref enum_def, _) => {
@@ -101,6 +104,7 @@ fn deserialize_body(
                 impl_generics,
                 ty,
                 enum_def,
+                &container_attrs,
             )
         }
         _ => cx.bug("expected ItemStruct or ItemEnum in #[derive(Deserialize)]")
@@ -114,6 +118,7 @@ fn deserialize_item_struct(
     impl_generics: &ast::Generics,
     ty: P<ast::Ty>,
     variant_data: &ast::VariantData,
+    container_attrs: &ContainerAttrs,
 ) -> P<ast::Expr> {
     match *variant_data {
         ast::VariantData::Unit(_) => {
@@ -158,6 +163,7 @@ fn deserialize_item_struct(
                 impl_generics,
                 ty,
                 fields,
+                container_attrs,
             )
         }
     }
@@ -461,6 +467,7 @@ fn deserialize_struct(
     impl_generics: &ast::Generics,
     ty: P<ast::Ty>,
     fields: &[ast::StructField],
+    container_attrs: &ContainerAttrs,
 ) -> P<ast::Expr> {
     let where_clause = &impl_generics.where_clause;
 
@@ -485,6 +492,7 @@ fn deserialize_struct(
         builder,
         type_path.clone(),
         fields,
+        container_attrs
     );
 
     let type_name = builder.expr().str(type_ident);
@@ -525,6 +533,7 @@ fn deserialize_item_enum(
     impl_generics: &ast::Generics,
     ty: P<ast::Ty>,
     enum_def: &EnumDef,
+    container_attrs: &ContainerAttrs
 ) -> P<ast::Expr> {
     let where_clause = &impl_generics.where_clause;
 
@@ -541,7 +550,8 @@ fn deserialize_item_enum(
                     .default()
                     .build()
             })
-            .collect()
+            .collect(),
+        container_attrs,
     );
 
     let variants_expr = builder.expr().addr_of().slice()
@@ -557,6 +567,12 @@ fn deserialize_item_enum(
         const VARIANTS: &'static [&'static str] = $variants_expr;
     ).unwrap();
 
+    let ignored_arm = if !container_attrs.deny_unknown_fields() {
+        Some(quote_arm!(cx, __Field::__ignore => { Err(::serde::de::Error::end_of_stream()) }))
+    } else {
+        None
+    };
+
     // Match arms to extract a variant from a string
     let variant_arms: Vec<_> = enum_def.variants.iter()
         .enumerate()
@@ -572,10 +588,12 @@ fn deserialize_item_enum(
                 impl_generics,
                 ty.clone(),
                 variant,
+                container_attrs,
             );
 
             quote_arm!(cx, $variant_name => { $expr })
         })
+        .chain(ignored_arm.into_iter())
         .collect();
 
     let (visitor_item, visitor_ty, visitor_expr, visitor_generics) =
@@ -616,6 +634,7 @@ fn deserialize_variant(
     generics: &ast::Generics,
     ty: P<ast::Ty>,
     variant: &ast::Variant,
+    container_attrs: &ContainerAttrs,
 ) -> P<ast::Expr> {
     let variant_ident = variant.node.name;
 
@@ -652,6 +671,7 @@ fn deserialize_variant(
                 generics,
                 ty,
                 fields,
+                container_attrs,
             )
         }
     }
@@ -708,6 +728,7 @@ fn deserialize_struct_variant(
     generics: &ast::Generics,
     ty: P<ast::Ty>,
     fields: &[ast::StructField],
+    container_attrs: &ContainerAttrs,
 ) -> P<ast::Expr> {
     let where_clause = &generics.where_clause;
 
@@ -728,6 +749,7 @@ fn deserialize_struct_variant(
         builder,
         type_path,
         fields,
+        container_attrs,
     );
 
     let (visitor_item, visitor_ty, visitor_expr, visitor_generics) =
@@ -771,12 +793,20 @@ fn deserialize_field_visitor(
     cx: &ExtCtxt,
     builder: &aster::AstBuilder,
     field_attrs: Vec<attr::FieldAttrs>,
+    container_attrs: &ContainerAttrs,
 ) -> Vec<P<ast::Item>> {
     // Create the field names for the fields.
     let field_idents: Vec<ast::Ident> = (0 .. field_attrs.len())
         .map(|i| builder.id(format!("__field{}", i)))
         .collect();
 
+    let ignore_variant = if !container_attrs.deny_unknown_fields() {
+        let skip_ident = builder.id("__ignore");
+        Some(builder.variant(skip_ident).unit())
+    } else {
+        None
+    };
+
     let field_enum = builder.item()
         .attr().allow(&["non_camel_case_types"])
         .enum_("__Field")
@@ -785,6 +815,7 @@ fn deserialize_field_visitor(
                 builder.variant(field_ident).unit()
             })
         )
+        .with_variants(ignore_variant.into_iter())
         .build();
 
     let index_field_arms: Vec<_> = field_idents.iter()
@@ -817,12 +848,18 @@ fn deserialize_field_visitor(
         })
         .collect();
 
+    let fallthrough_arm_expr = if !container_attrs.deny_unknown_fields() {
+        quote_expr!(cx, Ok(__Field::__ignore))
+    } else {
+        quote_expr!(cx, Err(::serde::de::Error::unknown_field(value)))
+    };
+
     let str_body = if formats.is_empty() {
         // No formats specific attributes, so no match on format required
         quote_expr!(cx,
             match value {
                 $default_field_arms
-                _ => { Err(::serde::de::Error::unknown_field(value)) }
+                _ => { $fallthrough_arm_expr }
             })
     } else {
         let field_arms: Vec<_> = formats.iter()
@@ -844,7 +881,7 @@ fn deserialize_field_visitor(
                     match value {
                         $arms
                         _ => {
-                            Err(::serde::de::Error::unknown_field(value))
+                            $fallthrough_arm_expr
                         }
                     }})
             })
@@ -855,7 +892,7 @@ fn deserialize_field_visitor(
                 $fmt_matches
                 _ => match value {
                     $default_field_arms
-                    _ => { Err(::serde::de::Error::unknown_field(value)) }
+                    _ => $fallthrough_arm_expr
                 }
             }
         )
@@ -920,11 +957,13 @@ fn deserialize_struct_visitor(
     builder: &aster::AstBuilder,
     struct_path: ast::Path,
     fields: &[ast::StructField],
+    container_attrs: &ContainerAttrs,
 ) -> (Vec<P<ast::Item>>, P<ast::Stmt>, P<ast::Expr>) {
     let field_visitor = deserialize_field_visitor(
         cx,
         builder,
         field::struct_field_attrs(cx, builder, fields),
+        container_attrs
     );
 
     let visit_map_expr = deserialize_map(
@@ -932,6 +971,7 @@ fn deserialize_struct_visitor(
         builder,
         struct_path,
         fields,
+        container_attrs,
     );
 
     let fields_expr = builder.expr().addr_of().slice()
@@ -958,6 +998,7 @@ fn deserialize_map(
     builder: &aster::AstBuilder,
     struct_path: ast::Path,
     fields: &[ast::StructField],
+    container_attrs: &ContainerAttrs,
 ) -> P<ast::Expr> {
     // Create the field names for the fields.
     let field_names: Vec<ast::Ident> = (0 .. fields.len())
@@ -969,6 +1010,16 @@ fn deserialize_map(
         .map(|field_name| quote_stmt!(cx, let mut $field_name = None;).unwrap())
         .collect();
 
+
+    // Visit ignored values to consume them
+    let ignored_arm = if !container_attrs.deny_unknown_fields() {
+        Some(quote_arm!(cx,
+            _ => { try!(visitor.visit_value::<::serde::de::impls::IgnoredAny>()); }
+        ))
+    } else {
+        None
+    };
+
     // Match arms to extract a value for a field.
     let value_arms: Vec<ast::Arm> = field_names.iter()
         .map(|field_name| {
@@ -978,6 +1029,7 @@ fn deserialize_map(
                 }
             )
         })
+        .chain(ignored_arm.into_iter())
         .collect();
 
     let extract_values: Vec<P<ast::Stmt>> = field_names.iter()

serde_codegen/src/field.rs

@@ -2,7 +2,7 @@ use syntax::ast;
 use syntax::ext::base::ExtCtxt;
 
 use aster;
-use attr::{FieldAttrs, FieldAttrsBuilder};
+use attr::{ContainerAttrs, ContainerAttrsBuilder, FieldAttrs, FieldAttrsBuilder};
 
 pub fn struct_field_attrs(
     _cx: &ExtCtxt,
@@ -15,3 +15,10 @@ pub fn struct_field_attrs(
         })
         .collect()
 }
+
+pub fn container_attrs(
+    _cx: &ExtCtxt,
+    container: &ast::Item,
+) -> ContainerAttrs {
+    ContainerAttrsBuilder::new().attrs(container.attrs()).build()
+}

serde_tests/tests/test_annotations.rs

@@ -1,6 +1,13 @@
 use std::default;
 
-use token::{Token, assert_tokens, assert_ser_tokens, assert_de_tokens};
+use token::{
+    Error,
+    Token,
+    assert_tokens,
+    assert_ser_tokens,
+    assert_de_tokens,
+    assert_de_tokens_error
+};
 
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 struct Default {
@@ -9,6 +16,12 @@ struct Default {
     a2: i32,
 }
 
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+struct DisallowUnknown {
+    a1: i32,
+}
+
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 struct Rename {
     a1: i32,
@@ -86,6 +99,59 @@ fn test_default() {
     );
 }
 
+#[test]
+fn test_ignore_unknown() {
+    // 'Default' allows unknown. Basic smoke test of ignore...
+    assert_de_tokens(
+        &Default { a1: 1, a2: 2},
+        vec![
+            Token::StructStart("Default", Some(5)),
+
+            Token::MapSep,
+            Token::Str("whoops1"),
+            Token::I32(2),
+
+            Token::MapSep,
+            Token::Str("a1"),
+            Token::I32(1),
+
+            Token::MapSep,
+            Token::Str("whoops2"),
+            Token::SeqStart(Some(1)),
+            Token::SeqSep,
+            Token::I32(2),
+            Token::SeqEnd,
+
+            Token::MapSep,
+            Token::Str("a2"),
+            Token::I32(2),
+
+            Token::MapSep,
+            Token::Str("whoops3"),
+            Token::I32(2),
+
+            Token::MapEnd,
+        ]
+    );
+
+    assert_de_tokens_error::<DisallowUnknown>(
+        vec![
+            Token::StructStart("DisallowUnknown", Some(2)),
+
+            Token::MapSep,
+            Token::Str("a1"),
+            Token::I32(1),
+
+            Token::MapSep,
+            Token::Str("whoops"),
+            Token::I32(2),
+
+            Token::MapEnd,
+        ],
+        Error::UnknownFieldError("whoops".to_owned())
+    );
+}
+
 #[test]
 fn test_rename() {
     assert_tokens(

serde_tests/tests/test_de.rs

@@ -7,7 +7,7 @@ use num::rational::Ratio;
 
 use serde::de::{Deserializer, Visitor};
 
-use token::{Token, assert_de_tokens};
+use token::{Error, Token, assert_de_tokens, assert_de_tokens_ignore};
 
 //////////////////////////////////////////////////////////////////////////
 
@@ -39,7 +39,11 @@ macro_rules! declare_test {
         #[test]
         fn $name() {
             $(
+                // Test ser/de roundtripping
                 assert_de_tokens(&$value, $tokens);
+
+                // Test that the tokens are ignorable
+                assert_de_tokens_ignore($tokens);
             )+
         }
     }

serde_tests/tests/token.rs

@@ -310,7 +310,7 @@ impl<'a, I> ser::Serializer for Serializer<I>
 //////////////////////////////////////////////////////////////////////////////
 
 #[derive(Clone, PartialEq, Debug)]
-enum Error {
+pub enum Error {
     SyntaxError,
     EndOfStreamError,
     UnknownFieldError(String),
@@ -644,7 +644,7 @@ impl<'a, I> de::MapVisitor for DeserializerMapVisitor<'a, I>
         match self.de.tokens.peek() {
             Some(&Token::MapSep) => {
                 self.de.tokens.next();
-                self.len = self.len.map(|len| len - 1);
+                self.len = self.len.map(|len| if len > 0 { len - 1} else { 0 });
                 Ok(Some(try!(de::Deserialize::deserialize(self.de))))
             }
             Some(&Token::MapEnd) => Ok(None),
@@ -799,6 +799,57 @@ pub fn assert_de_tokens<T>(value: &T, tokens: Vec<Token<'static>>)
     assert_eq!(de.tokens.next(), None);
 }
 
+// Expect an error deserializing tokens into a T
+pub fn assert_de_tokens_error<T>(tokens: Vec<Token<'static>>, error: Error)
+    where T: de::Deserialize + PartialEq + fmt::Debug,
+{
+    let mut de = Deserializer::new(tokens.into_iter());
+    let v: Result<T, Error> = de::Deserialize::deserialize(&mut de);
+    assert_eq!(v.as_ref(), Err(&error));
+}
+
+// Tests that the given token stream is ignorable when embedded in
+// an otherwise normal struct
+pub fn assert_de_tokens_ignore(ignorable_tokens: Vec<Token<'static>>) {
+    #[derive(PartialEq, Debug, Deserialize)]
+    struct IgnoreBase {
+        a: i32,
+    }
+
+    let expected = IgnoreBase{a: 1};
+
+    // Embed the tokens to be ignored in the normal token
+    // stream for an IgnoreBase type
+    let concated_tokens : Vec<Token<'static>> = vec![
+            Token::MapStart(Some(2)),
+                Token::MapSep,
+                Token::Str("a"),
+                Token::I32(1),
+
+                Token::MapSep,
+                Token::Str("ignored")
+        ]
+        .into_iter()
+        .chain(ignorable_tokens.into_iter())
+        .chain(vec![
+            Token::MapEnd,
+        ].into_iter())
+        .collect();
+
+    let mut de = Deserializer::new(concated_tokens.into_iter());
+    let v: Result<IgnoreBase, Error> = de::Deserialize::deserialize(&mut de);
+
+    // We run this test on every token stream for convenience, but
+    // some token streams don't make sense embedded as a map value,
+    // so we ignore those. SyntaxError is the real sign of trouble.
+    if let Err(Error::UnexpectedToken(_)) = v {
+        return;
+    }
+
+    assert_eq!(v.as_ref(), Ok(&expected));
+    assert_eq!(de.tokens.next(), None);
+}
+
 pub fn assert_tokens<T>(value: &T, tokens: Vec<Token<'static>>)
     where T: ser::Serialize + de::Deserialize + PartialEq + fmt::Debug,
 {