From ec87e6d80e53dac5ab6606b54f026e496550e5f4 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Wed, 23 Sep 2015 23:06:45 +0200 Subject: [PATCH 001/230] travis config --- .travis.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..a0c2fc5 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,16 @@ +language: python +python: + - "pypy" + - "2.7" + - "pypy3" + - "3.3" + - "3.4" + - "3.5" + +# Use fast travis build infrastructure explicitly +sudo: false + +# Installation installs dependencies +install: python setup.py install + +script: python -bb -E test_serpent.py From 2d699894faa3c460db599a61fd98121c2163042c Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Wed, 23 Sep 2015 23:09:06 +0200 Subject: [PATCH 002/230] travis config deps --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a0c2fc5..35648d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,8 @@ python: sudo: false # Installation installs dependencies -install: python setup.py install +install: + - pip install pytz + - pip install . -script: python -bb -E test_serpent.py +script: python -bb test_serpent.py From f71cf1c831180dcc3f09eb9495dca496349c2cb5 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Wed, 23 Sep 2015 23:19:06 +0200 Subject: [PATCH 003/230] readme converted to markdown --- README.txt => README.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) rename README.txt => README.md (83%) diff --git a/README.txt b/README.md similarity index 83% rename from README.txt rename to README.md index 4e08492..bd46b0d 100644 --- a/README.txt +++ b/README.md @@ -1,5 +1,10 @@ Serpent serialization library (Python/.NET/Java) ------------------------------------------------- +================================================ + +[![Build Status](https://travis-ci.org/irmen/Serpent.svg?branch=master)](https://travis-ci.org/irmen/Serpent) +[![Latest Version](https://img.shields.io/pypi/v/Serpent.svg)](https://pypi.python.org/pypi/Serpent/) + + Serpent provides ast.literal_eval() compatible object tree serialization. It serializes an object tree into bytes (utf-8 encoded string) that can be decoded and then passed as-is to ast.literal_eval() to rebuild it as the original object tree. @@ -9,7 +14,7 @@ As such it is safe to send serpent data to other machines over the network for i More info on Pypi: https://pypi.python.org/pypi/serpent Source code is on Github: https://github.com/irmen/Serpent -Copyright 2013, 2014 by Irmen de Jong (irmen@razorvine.net) +Copyright by Irmen de Jong (irmen@razorvine.net) This software is released under the MIT software license. This license, including disclaimer, is available in the 'LICENSE' file. @@ -61,18 +66,18 @@ to the object tree that is being serialized, and don't use the same serializer in different threads. Python 2.6 cannot deserialize complex numbers at all (limitation of -ast.literal_eval in 2.6). +``ast.literal_eval`` in 2.6). Because the serialized format is just valid Python source code, it can contain comments. Serpent does not add comments by itself apart from the single header line. -Set literals are not supported on python <3.2 (ast.literal_eval +Set literals are not supported on python <3.2 (``ast.literal_eval`` limitation). If you need Python < 3.2 compatibility, you'll have to use -set_literals=False when serializing. Since version 1.6 serpent chooses +``set_literals=False`` when serializing. Since version 1.6 serpent chooses this wisely for you by default, but you can still override it if needed. Floats +inf and -inf are handled via a trick, Float 'nan' cannot be handled -and is represented by the special value: {'__class__':'float','value':'nan'} +and is represented by the special value: ``{'__class__':'float','value':'nan'}`` We chose not to encode it as just the string 'NaN' because that could cause memory issues when used in multiplications. From 4edb50b5a1b11a76de1ddac4e3817f78f1ad402f Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Wed, 23 Sep 2015 23:20:08 +0200 Subject: [PATCH 004/230] readme converted to markdown --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bd46b0d..dbdddf2 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Serpent handles several special Python types to make life easier: - array.array other typecode --> list - Exception --> dict with some fields of the exception (message, args) - collections module types --> mostly equivalent primitive types or dict - - all other types --> dict with the __getstate__ or vars() of the object + - all other types --> dict with the ``__getstate__`` or ``vars()`` of the object Notes: From 1af167aacdc0aecc2400e0c23eddeaebc384bcb1 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 22 Dec 2015 14:34:08 +0100 Subject: [PATCH 005/230] sync python version number with release --- serpent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serpent.py b/serpent.py index 06b998b..988ab0f 100644 --- a/serpent.py +++ b/serpent.py @@ -67,7 +67,7 @@ import array import math -__version__ = "1.11" +__version__ = "1.12" __all__ = ["dump", "dumps", "load", "loads", "register_class", "unregister_class", "fix_nan"] can_use_set_literals = sys.version_info >= (3, 2) # check if we can use set literals From 838ae3a636fcf7e56cd49177af8e44ef34930ce2 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Wed, 27 Jan 2016 01:51:06 +0100 Subject: [PATCH 006/230] nunit command update --- dotnet/test-dotnet.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/dotnet/test-dotnet.sh b/dotnet/test-dotnet.sh index c41e7d0..369e4ca 100755 --- a/dotnet/test-dotnet.sh +++ b/dotnet/test-dotnet.sh @@ -3,8 +3,4 @@ echo "Building..." . ./build-dotnet-mono.sh echo "Running tests" - -# note: nunit-2.6.2 crashes on Mono. Stick with 2.6.1 for the time being. -NUNIT="mono ${MONO_OPTIONS} ${HOME}/Projects/NUnit-2.6.1/bin/nunit-console.exe" - -${NUNIT} -framework:4.0 -noshadow -nothread Serpent.Test/bin/Release/Razorvine.Serpent.Test.dll +nunit-console -noshadow Serpent.Test/bin/Release/Razorvine.Serpent.Test.dll From 535811d6d84479a5fb460e7cda3267c9da090bad Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Fri, 10 Jun 2016 18:22:45 +0200 Subject: [PATCH 007/230] fixed a few pylint issues --- .pylintrc | 7 +++ .../java/net/razorvine/serpent/ast/Ast.java | 6 +-- serpent.py | 53 +++++++++---------- 3 files changed, 36 insertions(+), 30 deletions(-) create mode 100644 .pylintrc diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..0ad4b77 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,7 @@ +[MESSAGES CONTROL] +disable=missing-docstring + +[BASIC] +include-naming-hint=yes +max-line-length=120 +good-names=i,j,k,x,y,z,t,ex,_ diff --git a/java/src/main/java/net/razorvine/serpent/ast/Ast.java b/java/src/main/java/net/razorvine/serpent/ast/Ast.java index d971069..1b02565 100644 --- a/java/src/main/java/net/razorvine/serpent/ast/Ast.java +++ b/java/src/main/java/net/razorvine/serpent/ast/Ast.java @@ -10,9 +10,9 @@ import net.razorvine.serpent.IDictToInstance; import net.razorvine.serpent.ObjectifyVisitor; -/// -/// Abstract syntax tree for the literal expression. This is what the parser returns. -/// +/** + * Abstract syntax tree for the literal expression. This is what the parser returns. + */ public class Ast { public INode root; diff --git a/serpent.py b/serpent.py index 988ab0f..9c0e36e 100644 --- a/serpent.py +++ b/serpent.py @@ -67,8 +67,8 @@ import array import math -__version__ = "1.12" -__all__ = ["dump", "dumps", "load", "loads", "register_class", "unregister_class", "fix_nan"] +__version__ = "1.13" +__all__ = ["dump", "dumps", "load", "loads", "register_class", "unregister_class"] can_use_set_literals = sys.version_info >= (3, 2) # check if we can use set literals @@ -193,11 +193,11 @@ def _repr(obj): } if sys.version_info >= (3, 0): - _translate_types.update({ - collections.UserDict: dict, - collections.UserList: list, - collections.UserString: str - }) + _translate_types.update({ + collections.UserDict: dict, + collections.UserList: list, + collections.UserString: str + }) # do some dynamic changes to the types configuration if needed if bytes is str: @@ -275,20 +275,19 @@ def _serialize(self, obj, out, level): if isinstance(obj, clazz): special_classes[clazz](obj, self, out, level) return - else: - # serialize dispatch - try: - f = self.dispatch[t] - except KeyError: - # walk the MRO until we find a base class we recognise - for type_ in t.__mro__: - if type_ in self.dispatch: - f = self.dispatch[type_] - break - else: - # fall back to the default class serializer - f = Serializer.ser_default_class - f(self, obj, out, level) + # serialize dispatch + try: + func = self.dispatch[t] + except KeyError: + # walk the MRO until we find a base class we recognise + for type_ in t.__mro__: + if type_ in self.dispatch: + func = self.dispatch[type_] + break + else: + # fall back to the default class serializer + func = Serializer.ser_default_class + func(self, obj, out, level) def ser_builtins_str(self, str_obj, out, level): # special case str, for IronPython where str==unicode and repr() yields undesired result @@ -415,20 +414,20 @@ def ser_builtins_dict(self, dict_obj, out, level): sorted_items = sorted(dict_items) except TypeError: # can occur when elements can't be ordered (Python 3.x) sorted_items = dict_items - for k, v in sorted_items: + for key, value in sorted_items: append(indent_chars_inside) - serialize(k, out, level + 1) + serialize(key, out, level + 1) append(b": ") - serialize(v, out, level + 1) + serialize(value, out, level + 1) append(b",\n") del out[-1] # remove last ,\n append(b"\n" + indent_chars + b"}") else: append(b"{") - for k, v in dict_obj.items(): - serialize(k, out, level + 1) + for key, value in dict_obj.items(): + serialize(key, out, level + 1) append(b":") - serialize(v, out, level + 1) + serialize(value, out, level + 1) append(b",") if dict_obj: del out[-1] # remove the last , From 9a42c25826b9b1644dae812de485743de3d30d78 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sun, 12 Jun 2016 11:45:03 +0200 Subject: [PATCH 008/230] py36 added to tox --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 4c35bad..8168830 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=py26,py27,py32,py33,py34,py35,pypy +envlist=py26,py27,py32,py33,py34,py35,py36,pypy [testenv] commands=python -bb -E test_serpent.py From c8877568bdf523cb7d42492ff97feb2b6c252ea6 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sun, 12 Jun 2016 11:46:49 +0200 Subject: [PATCH 009/230] py36 added to travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 35648d6..02c01bd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ python: - "3.3" - "3.4" - "3.5" + - "3.6" # Use fast travis build infrastructure explicitly sudo: false From 9c92bd7bbb257626f3900225558e6132e6262437 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sun, 12 Jun 2016 11:51:35 +0200 Subject: [PATCH 010/230] 3.6 removed from travis, not installed there yet --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 02c01bd..35648d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,6 @@ python: - "3.3" - "3.4" - "3.5" - - "3.6" # Use fast travis build infrastructure explicitly sudo: false From 6c22018b4d6dad72ce05e853e9e09ee35a9fa3b5 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Mon, 20 Jun 2016 23:28:02 +0200 Subject: [PATCH 011/230] able to serialize dict_keys/dict_values/dict_items directly into lists. Fixes #14 --- serpent.py | 10 +++++++++- test_serpent.py | 12 ++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/serpent.py b/serpent.py index 9c0e36e..cd6fbae 100644 --- a/serpent.py +++ b/serpent.py @@ -118,7 +118,15 @@ def _ser_OrderedDict(obj, serializer, outputstream, indentlevel): serializer._serialize(obj, outputstream, indentlevel) -_special_classes_registry = {} +def _ser_DictView(obj, serializer, outputstream, indentlevel): + serializer.ser_builtins_list(obj, outputstream, indentlevel) + + +_special_classes_registry = { + collections.KeysView: _ser_DictView, + collections.ValuesView: _ser_DictView, + collections.ItemsView: _ser_DictView +} if sys.version_info >= (2, 7): _special_classes_registry[collections.OrderedDict] = _ser_OrderedDict diff --git a/test_serpent.py b/test_serpent.py index ccc4f64..b6781df 100644 --- a/test_serpent.py +++ b/test_serpent.py @@ -313,6 +313,18 @@ def test_dict(self): self.assertEqual(ord("{"), data[0]) self.assertEqual(ord("}"), data[-1]) + def test_dict_iters(self): + data = {"john": 22, "sophie": 34, "bob": 26} + ser = serpent.loads(serpent.dumps(data.keys())) + self.assertIsInstance(ser, list) + self.assertEqual(["bob", "john", "sophie"], sorted(ser)) + ser = serpent.loads(serpent.dumps(data.values())) + self.assertIsInstance(ser, list) + self.assertEqual([22, 26, 34], sorted(ser)) + ser = serpent.loads(serpent.dumps(data.items())) + self.assertIsInstance(ser, list) + self.assertEqual([("bob", 26), ("john", 22), ("sophie", 34)], sorted(ser)) + def test_list(self): ser = serpent.dumps([]) data = strip_header(ser) From 03acc7d6460bbd2770dcb9ef4e43432676c14cc1 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Thu, 8 Sep 2016 23:11:03 +0200 Subject: [PATCH 012/230] extra unittest to make sure subclasses are also accepted for custom serializers --- test_serpent.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test_serpent.py b/test_serpent.py index b6781df..933985c 100644 --- a/test_serpent.py +++ b/test_serpent.py @@ -714,6 +714,11 @@ def __init__(self, name, value): def __getstate__(self): return ("bogus", "state") +class BaseClass(object): + pass +class SubClass(BaseClass): + pass + class TestCustomClasses(unittest.TestCase): def testCustomClass(self): @@ -736,6 +741,15 @@ def something_serializer(obj, serializer, stream, level): x = serpent.loads(d) self.assertEqual(("bogus", "state"), x) + def testSubclass(self): + def custom_serializer(obj, serializer, stream, level): + serializer._serialize("[(sub)class=%s]" % type(obj), stream, level) + serpent.register_class(BaseClass, custom_serializer) + s = SubClass() + d = serpent.dumps(s) + x = serpent.loads(d) + self.assertEqual("[(sub)class=]", x) + def testUUID(self): uid = uuid.uuid4() string_uid = str(uid) From aac2cf1292f373f5da7a13eba8d7aca36a88320a Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Fri, 9 Sep 2016 00:14:40 +0200 Subject: [PATCH 013/230] custom class converters now apply to inheritance tree rather than just one specific class. Bump version to 1.13. --- dotnet/Serpent.Test/SerializeTest.cs | 59 +++++++++++++++++++ dotnet/Serpent/Properties/AssemblyInfo.cs | 4 +- dotnet/Serpent/Serializer.cs | 20 ++++++- dotnet/Serpent/Serpent.nuspec | 6 +- dotnet/Serpent/Version.cs | 4 +- .../net/razorvine/serpent/LibraryVersion.java | 2 +- .../net/razorvine/serpent/Serializer.java | 21 ++++++- .../net/razorvine/serpent/package-info.java | 2 +- .../razorvine/serpent/test/SerializeTest.java | 52 ++++++++++++++++ 9 files changed, 158 insertions(+), 12 deletions(-) diff --git a/dotnet/Serpent.Test/SerializeTest.cs b/dotnet/Serpent.Test/SerializeTest.cs index 3b0b693..f0cf427 100644 --- a/dotnet/Serpent.Test/SerializeTest.cs +++ b/dotnet/Serpent.Test/SerializeTest.cs @@ -611,6 +611,65 @@ public void TestEnum() byte[] ser = strip_header(serpent.Serialize(e)); Assert.AreEqual("'Jarjar'", S(ser)); } + + + interface IBaseInterface {}; + interface ISubInterface : IBaseInterface {}; + class BaseClassWithInterface : IBaseInterface {}; + class SubClassWithInterface : BaseClassWithInterface, ISubInterface {}; + class BaseClass {}; + class SubClass : BaseClass {}; + abstract class AbstractBaseClass {}; + class ConcreteSubClass : AbstractBaseClass {}; + + protected IDictionary AnyClassSerializer(object arg) + { + IDictionary result = new Hashtable(); + result["(SUB)CLASS"] = arg.GetType().Name; + return result; + } + + [Test] + public void testAbstractBaseClassHierarchyPickler() + { + ConcreteSubClass c = new ConcreteSubClass(); + Serializer serpent = new Serializer(); + try { + serpent.Serialize(c); + Assert.Fail("should crash"); + } catch (SerializationException x) { + Assert.IsTrue(x.Message.Contains("not serializable")); + } + + Serializer.RegisterClass(typeof(AbstractBaseClass), AnyClassSerializer); + byte[] data = serpent.Serialize(c); + Assert.AreEqual("{'(SUB)CLASS':'ConcreteSubClass'}", S(strip_header(data))); + } + + [Test] + public void testInterfaceHierarchyPickler() + { + BaseClassWithInterface b = new BaseClassWithInterface(); + SubClassWithInterface sub = new SubClassWithInterface(); + Serializer serpent = new Serializer(); + try { + serpent.Serialize(b); + Assert.Fail("should crash"); + } catch (SerializationException x) { + Assert.IsTrue(x.Message.Contains("not serializable")); + } + try { + serpent.Serialize(sub); + Assert.Fail("should crash"); + } catch (SerializationException x) { + Assert.IsTrue(x.Message.Contains("not serializable")); + } + Serializer.RegisterClass(typeof(IBaseInterface), AnyClassSerializer); + byte[] data = serpent.Serialize(b); + Assert.AreEqual("{'(SUB)CLASS':'BaseClassWithInterface'}", S(strip_header(data))); + data = serpent.Serialize(sub); + Assert.AreEqual("{'(SUB)CLASS':'SubClassWithInterface'}", S(strip_header(data))); + } } [Serializable] diff --git a/dotnet/Serpent/Properties/AssemblyInfo.cs b/dotnet/Serpent/Properties/AssemblyInfo.cs index 4187641..d58e8ce 100644 --- a/dotnet/Serpent/Properties/AssemblyInfo.cs +++ b/dotnet/Serpent/Properties/AssemblyInfo.cs @@ -36,5 +36,5 @@ decoded there (using ast.literal_eval()). It can ofcourse also deserialize // // You can specify all the values or you can use the default the Revision and // Build Numbers by using the '*' as shown below: -[assembly: AssemblyVersion("1.12.0.*")] -[assembly: AssemblyFileVersion("1.12.0.0")] +[assembly: AssemblyVersion("1.13.0.*")] +[assembly: AssemblyFileVersion("1.13.0.0")] diff --git a/dotnet/Serpent/Serializer.cs b/dotnet/Serpent/Serializer.cs index ec2d2c8..f2a7cee 100644 --- a/dotnet/Serpent/Serializer.cs +++ b/dotnet/Serpent/Serializer.cs @@ -446,8 +446,7 @@ protected void Serialize_class(object obj, TextWriter tw, int level) Type obj_type = obj.GetType(); IDictionary dict; - Func converter = null; - classToDictRegistry.TryGetValue(obj_type, out converter); + Func converter = GetCustomConverter(obj_type); if(converter!=null) { @@ -487,5 +486,22 @@ protected void Serialize_class(object obj, TextWriter tw, int level) Serialize_dict(dict, tw, level); } + + protected Func GetCustomConverter(Type obj_type) + { + Func converter; + if(classToDictRegistry.TryGetValue(obj_type, out converter)) + return converter; // exact match + + // check if there's a custom converter registered for an interface or abstract base class + // that this object implements or inherits from. + foreach(var x in classToDictRegistry) { + if(x.Key.IsAssignableFrom(obj_type)) { + return x.Value; + } + } + + return null; + } } } diff --git a/dotnet/Serpent/Serpent.nuspec b/dotnet/Serpent/Serpent.nuspec index 1c6d989..d4a8092 100644 --- a/dotnet/Serpent/Serpent.nuspec +++ b/dotnet/Serpent/Serpent.nuspec @@ -2,7 +2,7 @@ Razorvine.Serpent - 1.12.0.0 + 1.13.0.0 Razorvine.Serpent Irmen de Jong Irmen de Jong @@ -18,8 +18,8 @@ such a Python expression itself, back into the equivalent .NET datatypes. More info for the Python version is on Pypi: https://pypi.python.org/pypi/serpent Serpent Python literal expression serialization - number parsing performance on large arrays greatly improved - Copyright 2015 + custom converters now work for inheritance tree rather than just one specific class + Copyright 2016 serialization python pyro diff --git a/dotnet/Serpent/Version.cs b/dotnet/Serpent/Version.cs index 6c1d6fe..3fa80d9 100644 --- a/dotnet/Serpent/Version.cs +++ b/dotnet/Serpent/Version.cs @@ -2,7 +2,7 @@ /// Serpent, a Python literal expression serializer/deserializer /// (a.k.a. Python's ast.literal_eval in .NET) /// -/// Copyright 2014, Irmen de Jong (irmen@razorvine.net) +/// Copyright: Irmen de Jong (irmen@razorvine.net) /// Software license: "MIT software license". See http://opensource.org/licenses/MIT /// @@ -10,6 +10,6 @@ namespace Razorvine.Serpent { public static class LibraryVersion { - public static string Version = "1.12"; + public static string Version = "1.13"; } } diff --git a/java/src/main/java/net/razorvine/serpent/LibraryVersion.java b/java/src/main/java/net/razorvine/serpent/LibraryVersion.java index df89299..1ca1064 100644 --- a/java/src/main/java/net/razorvine/serpent/LibraryVersion.java +++ b/java/src/main/java/net/razorvine/serpent/LibraryVersion.java @@ -8,5 +8,5 @@ package net.razorvine.serpent; public final class LibraryVersion { - public static final String VERSION = "1.12"; + public static final String VERSION = "1.13"; } diff --git a/java/src/main/java/net/razorvine/serpent/Serializer.java b/java/src/main/java/net/razorvine/serpent/Serializer.java index a31ddf2..21bc0b4 100644 --- a/java/src/main/java/net/razorvine/serpent/Serializer.java +++ b/java/src/main/java/net/razorvine/serpent/Serializer.java @@ -19,9 +19,11 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.*; +import java.util.Map.Entry; import javax.xml.bind.DatatypeConverter; + /** * Serialize an object tree to a byte stream. * It is not thread-safe: make sure you're not making changes to the object tree that is being serialized. @@ -403,7 +405,7 @@ protected boolean isBoxed(Class type) protected void serialize_class(Object obj, PrintWriter p, int level) { Map map; - IClassSerializer converter=classToDictRegistry.get(obj.getClass()); + IClassSerializer converter=getCustomConverter(obj.getClass()); if(null!=converter) { map = converter.convert(obj); @@ -447,6 +449,23 @@ protected void serialize_class(Object obj, PrintWriter p, int level) serialize_dict(map, p, level); } + protected IClassSerializer getCustomConverter(Class type) { + IClassSerializer converter = classToDictRegistry.get(type.getClass()); + if(converter!=null) { + return converter; // exact match + } + + // check if there's a custom pickler registered for an interface or abstract base class + // that this object implements or inherits from. + for(Entry, IClassSerializer> x: classToDictRegistry.entrySet()) { + if(x.getKey().isAssignableFrom(type)) { + return x.getValue(); + } + } + + return null; + } + protected void serialize_primitive(Object obj, PrintWriter p, int level) { if(obj instanceof Boolean || obj.getClass()==Boolean.TYPE) diff --git a/java/src/main/java/net/razorvine/serpent/package-info.java b/java/src/main/java/net/razorvine/serpent/package-info.java index 39d0e64..7852286 100644 --- a/java/src/main/java/net/razorvine/serpent/package-info.java +++ b/java/src/main/java/net/razorvine/serpent/package-info.java @@ -3,7 +3,7 @@ * (a.k.a. Python's ast.literal_eval in Java) * Software license: "MIT software license". See http://opensource.org/licenses/MIT * @author Irmen de Jong (irmen@razorvine.net) - * @version 1.12 + * @version 1.13 */ package net.razorvine.serpent; diff --git a/java/src/test/java/net/razorvine/serpent/test/SerializeTest.java b/java/src/test/java/net/razorvine/serpent/test/SerializeTest.java index c164a9e..f9e500d 100644 --- a/java/src/test/java/net/razorvine/serpent/test/SerializeTest.java +++ b/java/src/test/java/net/razorvine/serpent/test/SerializeTest.java @@ -9,6 +9,7 @@ import static org.junit.Assert.*; +import java.io.Serializable; import java.io.UnsupportedEncodingException; import java.math.BigDecimal; import java.math.BigInteger; @@ -18,6 +19,7 @@ import net.razorvine.serpent.IClassSerializer; import net.razorvine.serpent.Parser; import net.razorvine.serpent.Serializer; + import org.junit.Test; public class SerializeTest { @@ -613,4 +615,54 @@ public void testSorting() ser = strip_header(serpent.serialize(data5)); assertEquals("{\n 'a',\n 'b',\n 'c',\n 'x',\n 'y',\n 'z'\n}", S(ser)); } + + interface IBaseInterface {}; + interface ISubInterface extends IBaseInterface {}; + class BaseClassWithInterface implements IBaseInterface, Serializable {}; + class SubClassWithInterface extends BaseClassWithInterface implements ISubInterface, Serializable {}; + class BaseClass implements Serializable {}; + class SubClass extends BaseClass implements Serializable {}; + abstract class AbstractBaseClass {}; + class ConcreteSubClass extends AbstractBaseClass implements Serializable {}; + + class AnyClassConverter implements IClassSerializer + { + @Override + public Map convert(Object obj) { + Map result = new HashMap(); + result.put("(SUB)CLASS", obj.getClass().getSimpleName()); + return result; + } + } + + @Test + public void testAbstractBaseClassHierarchyPickler() + { + ConcreteSubClass c = new ConcreteSubClass(); + Serializer serpent=new Serializer(); + byte[] data = serpent.serialize(c); + assertEquals("{'__class__':'ConcreteSubClass'}", S(strip_header(data))); // the default serializer + + Serializer.registerClass(AbstractBaseClass.class, new AnyClassConverter()); + data = serpent.serialize(c); + assertEquals("{'(SUB)CLASS':'ConcreteSubClass'}", S(strip_header(data))); // custom serializer + } + + @Test + public void testInterfaceHierarchyPickler() + { + BaseClassWithInterface b = new BaseClassWithInterface(); + SubClassWithInterface sub = new SubClassWithInterface(); + Serializer serpent=new Serializer(); + byte[] data = serpent.serialize(b); + assertEquals("{'__class__':'BaseClassWithInterface'}", S(strip_header(data))); // the default serializer + data = serpent.serialize(sub); + assertEquals("{'__class__':'SubClassWithInterface'}", S(strip_header(data))); // the default serializer + + Serializer.registerClass(IBaseInterface.class, new AnyClassConverter()); + data = serpent.serialize(b); + assertEquals("{'(SUB)CLASS':'BaseClassWithInterface'}", S(strip_header(data))); // custom serializer + data = serpent.serialize(sub); + assertEquals("{'(SUB)CLASS':'SubClassWithInterface'}", S(strip_header(data))); // custom serializer + } } From 442f1b2fcf35a469aacad3842097411d3671d8af Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Fri, 9 Sep 2016 00:37:32 +0200 Subject: [PATCH 014/230] [maven-release-plugin] prepare release serpent-1.13 --- java/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index b24b20d..61acb09 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -9,7 +9,7 @@ net.razorvine serpent - 1.13-SNAPSHOT + 1.13 jar serpent @@ -67,7 +67,7 @@ It is an alternative to JSON to provide easy data integration between Java and P https://github.com/irmen/Serpent scm:git:https://github.com/irmen/Serpent.git scm:git:https://github.com/irmen/Serpent.git - HEAD + serpent-1.13 Github From d47c903e37233639279d9da8c08d44c86ada8687 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Fri, 9 Sep 2016 00:37:51 +0200 Subject: [PATCH 015/230] [maven-release-plugin] prepare for next development iteration --- java/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 61acb09..81d451e 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -9,7 +9,7 @@ net.razorvine serpent - 1.13 + 1.14-SNAPSHOT jar serpent @@ -67,7 +67,7 @@ It is an alternative to JSON to provide easy data integration between Java and P https://github.com/irmen/Serpent scm:git:https://github.com/irmen/Serpent.git scm:git:https://github.com/irmen/Serpent.git - serpent-1.13 + HEAD Github From b5b42f1b6b06d156172167b704eb46c50c363e12 Mon Sep 17 00:00:00 2001 From: Matthew Moisen Date: Wed, 14 Sep 2016 19:57:31 +0000 Subject: [PATCH 016/230] add support for datetime.date --- serpent.py | 6 +++++- test_serpent.py | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/serpent.py b/serpent.py index cd6fbae..fd2603e 100644 --- a/serpent.py +++ b/serpent.py @@ -489,7 +489,11 @@ def ser_decimal_Decimal(self, decimal_obj, out, level): def ser_datetime_datetime(self, datetime_obj, out, level): self._serialize(datetime_obj.isoformat(), out, level) dispatch[datetime.datetime] = ser_datetime_datetime - + + def ser_datetime_date(self, date_obj, out, level): + self._serialize(date_obj.isoformat(), out, level) + dispatch[datetime.date] = ser_datetime_date + if os.name == "java" or sys.version_info < (2, 7): # jython bug http://bugs.jython.org/issue2010 def ser_datetime_timedelta(self, timedelta_obj, out, level): secs = ((timedelta_obj.days * 86400 + timedelta_obj.seconds) * 10 ** 6 + timedelta_obj.microseconds) / 10 ** 6 diff --git a/test_serpent.py b/test_serpent.py index 933985c..448f320 100644 --- a/test_serpent.py +++ b/test_serpent.py @@ -508,6 +508,9 @@ def test_time(self): ser = serpent.dumps(datetime.datetime(2013, 1, 20, 23, 59, 45, 999888)) data = strip_header(ser) self.assertEqual(b"'2013-01-20T23:59:45.999888'", data) + ser = serpent.dumps(datetime.date(2013, 1, 20)) + data = strip_header(ser) + self.assertEqual(b"'2013-01-20'", data) ser = serpent.dumps(datetime.time(23, 59, 45, 999888)) data = strip_header(ser) self.assertEqual(b"'23:59:45.999888'", data) @@ -564,6 +567,7 @@ def setUp(self): "exc": ZeroDivisionError("fault"), "dates": [ datetime.datetime.now(), + datetime.date.today(), datetime.time(23, 59, 45, 999888), datetime.timedelta(seconds=500) ], From d6e25d02b77a012aac7c1eb0318d1fc821a41de0 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Fri, 16 Sep 2016 21:50:13 +0200 Subject: [PATCH 017/230] version bump --- README.md | 2 +- serpent.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index dbdddf2..d166549 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Serpent handles several special Python types to make life easier: - str --> promoted to unicode (see below why this is) - bytes, bytearrays, memoryview, buffer --> string, base-64 (you'll have to manually un-base64 them though) - - uuid.UUID, datetime.{datetime, time, timespan} --> appropriate string/number + - uuid.UUID, datetime.{datetime, date, time, timespan} --> appropriate string/number - decimal.Decimal --> string (to not lose precision) - array.array typecode 'c'/'u' --> string/unicode - array.array other typecode --> list diff --git a/serpent.py b/serpent.py index fd2603e..067deda 100644 --- a/serpent.py +++ b/serpent.py @@ -14,7 +14,7 @@ - str --> promoted to unicode (see below why this is) - bytes, bytearrays, memoryview, buffer --> string, base-64 (you'll have to manually un-base64 them though) - - uuid.UUID, datetime.{datetime, time, timespan} --> appropriate string/number + - uuid.UUID, datetime.{datetime, date, time, timespan} --> appropriate string/number - decimal.Decimal --> string (to not lose precision) - array.array typecode 'c'/'u' --> string/unicode - array.array other typecode --> list @@ -48,7 +48,7 @@ We chose not to encode it as just the string 'NaN' because that could cause memory issues when used in multiplications. -Copyright 2013, 2014 by Irmen de Jong (irmen@razorvine.net) +Copyright by Irmen de Jong (irmen@razorvine.net) Software license: "MIT software license". See http://opensource.org/licenses/MIT """ @@ -67,7 +67,7 @@ import array import math -__version__ = "1.13" +__version__ = "1.14" __all__ = ["dump", "dumps", "load", "loads", "register_class", "unregister_class"] can_use_set_literals = sys.version_info >= (3, 2) # check if we can use set literals From 534524e3dbeb6a35053022e0ee565d68d7e3d415 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Mon, 10 Oct 2016 02:29:07 +0200 Subject: [PATCH 018/230] require java7 --- java/.classpath | 6 +----- java/pom.xml | 4 ++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/java/.classpath b/java/.classpath index 921c731..2fa8e78 100644 --- a/java/.classpath +++ b/java/.classpath @@ -13,15 +13,11 @@ - - - - - + diff --git a/java/pom.xml b/java/pom.xml index 81d451e..0a9d9bb 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -26,8 +26,8 @@ maven-compiler-plugin 3.3 - 1.6 - 1.6 + 1.7 + 1.7 From 4c2a55994218b5f47af0e32a45207cf8875add8d Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Mon, 10 Oct 2016 22:47:25 +0200 Subject: [PATCH 019/230] added tobytes utility function. version 1.15 --- README.md | 2 +- serpent.py | 27 ++++++++++++++++++++++++--- setup.py | 3 ++- test_serpent.py | 32 +++++++++++++++++++++++++++++++- 4 files changed, 58 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d166549..bb6f31b 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ Serpent handles several special Python types to make life easier: - str --> promoted to unicode (see below why this is) - bytes, bytearrays, memoryview, buffer --> string, base-64 - (you'll have to manually un-base64 them though) + (you'll have to manually un-base64 them though. Can use serpent.tobytes function) - uuid.UUID, datetime.{datetime, date, time, timespan} --> appropriate string/number - decimal.Decimal --> string (to not lose precision) - array.array typecode 'c'/'u' --> string/unicode diff --git a/serpent.py b/serpent.py index 067deda..4af577d 100644 --- a/serpent.py +++ b/serpent.py @@ -67,8 +67,8 @@ import array import math -__version__ = "1.14" -__all__ = ["dump", "dumps", "load", "loads", "register_class", "unregister_class"] +__version__ = "1.15" +__all__ = ["dump", "dumps", "load", "loads", "register_class", "unregister_class", "tobytes"] can_use_set_literals = sys.version_info >= (3, 2) # check if we can use set literals @@ -147,7 +147,10 @@ def register_class(clazz, serializer): class BytesWrapper(object): - """Wrapper for bytes, bytearray etc. to make them appear as base-64 encoded data.""" + """ + Wrapper for bytes, bytearray etc. to make them appear as base-64 encoded data. + You can use the tobytes utility function to decode this back into the actual bytes (or do it manually) + """ def __init__(self, data): self.data = data @@ -207,17 +210,35 @@ def _repr(obj): collections.UserString: str }) +_bytes_types = [bytes, bytearray, memoryview] + # do some dynamic changes to the types configuration if needed if bytes is str: del _translate_types[bytes] if hasattr(types, "BufferType"): _translate_types[types.BufferType] = BytesWrapper.from_buffer + _bytes_types.append(buffer) try: _translate_types[memoryview] = BytesWrapper.from_memoryview except NameError: pass if sys.platform == "cli": _repr_types.remove(str) # IronPython needs special str treatment, otherwise it treats unicode wrong +_bytes_types = tuple(_bytes_types) + + +def tobytes(obj): + """ + Utility function to convert obj back to actual bytes if it is a serpent-encoded bytes dictionary + (a dict with base-64 encoded 'data' in it and 'encoding'='base64'). + If obj is already bytes or a byte-like type, return obj unmodified. + Will raise TypeError if obj is none of the above. + """ + if isinstance(obj, _bytes_types): + return obj + if isinstance(obj, dict) and "data" in obj and obj.get("encoding") == "base64": + return base64.b64decode(obj["data"]) + raise TypeError("argument is neither bytes nor serpent base64 encoded bytes dict") class Serializer(object): diff --git a/setup.py b/setup.py index 21ebd84..0f67b08 100755 --- a/setup.py +++ b/setup.py @@ -43,7 +43,8 @@ - it serializes directly to bytes (utf-8 encoded), instead of a string, so it can immediately be saved to a file or sent over a socket - it encodes byte-types as base-64 instead of inefficient escaping notation that repr would use (this does mean you have - to base-64 decode these strings manually on the receiving side to get your bytes back) + to base-64 decode these strings manually on the receiving side to get your bytes back. + You can use the serpent.tobytes utility function for this.) - it contains a few custom serializers for several additional Python types such as uuid, datetime, array and decimal - it tries to serialize unrecognised types as a dict (you can control this with __getstate__ on your own types) - it can create a pretty-printed (indented) output for readability purposes diff --git a/test_serpent.py b/test_serpent.py index 448f320..478c433 100644 --- a/test_serpent.py +++ b/test_serpent.py @@ -18,6 +18,7 @@ import threading import time import collections +import types if sys.version_info < (2, 7): import unittest2 as unittest @@ -552,7 +553,36 @@ def test_weird_floats(self): ser = strip_header(serpent.dumps(values)) self.assertEqual(b"[1e30000,-1e30000,{'__class__':'float','value':'nan'},(1e30000+4.0j)]", ser) - + def test_tobytes(self): + obj = b"test" + self.assertIs(obj, serpent.tobytes(obj)) + obj = memoryview(b"test") + self.assertIs(obj, serpent.tobytes(obj)) + obj = bytearray(b"test") + self.assertIs(obj, serpent.tobytes(obj)) + if hasattr(types, "BufferType"): + obj = buffer(b"test") + self.assertIs(obj, serpent.tobytes(obj)) + ser = {'data': 'dGVzdA==', 'encoding': 'base64'} + out = serpent.tobytes(ser) + self.assertEqual(b"test", out) + if sys.platform == 'cli': + self.assertIsInstance(out, str) # ironpython base64 decodes into str type.... + else: + self.assertIsInstance(out, bytes) + with self.assertRaises(TypeError): + serpent.tobytes({'@@@data': 'dGVzdA==', 'encoding': 'base64'}) + with self.assertRaises(TypeError): + serpent.tobytes({'data': 'dGVzdA==', '@@@encoding': 'base64'}) + with self.assertRaises(TypeError): + serpent.tobytes({'data': 'dGVzdA==', 'encoding': 'base99'}) + with self.assertRaises(TypeError): + serpent.tobytes({}) + with self.assertRaises(TypeError): + serpent.tobytes(42) + + +@unittest.skip("no performance tests in default test suite") class TestSpeed(unittest.TestCase): def setUp(self): self.data = { From 76884582ddecc3c530b45415dae131f3d4aaaf8c Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Mon, 10 Oct 2016 23:17:13 +0200 Subject: [PATCH 020/230] dotnet added Serializer.ToBytes utility function to decode a serpent base-64 encoded byte array back into the actual byte array --- .gitignore | 46 ++++++++++++++++++++++++++++ dotnet/Serpent.Test/SerializeTest.cs | 20 ++++++++++++ dotnet/Serpent/Parser.cs | 2 +- dotnet/Serpent/Serializer.cs | 28 +++++++++++++++++ dotnet/Serpent/Serpent.nuspec | 5 ++- 5 files changed, 99 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index f77f689..56df97e 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,52 @@ develop-eggs lib lib64 +# Build Folders (you can keep bin if you'd like, to store dlls and pdbs) +[Bb]in/ +[Oo]bj/ +build +target + +# mstest test results +TestResults +TEST-*.xml +TestResult.xml + +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. + +# User-specific files +*.suo +*.user +*.sln.docstates + +# Build results +[Dd]ebug/ +[Rr]elease/ +x64/ +*_i.c +*_p.c +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.log +*.vspscc +*.vssscc +.builds +*.nupkg + + + # Installer logs pip-log.txt diff --git a/dotnet/Serpent.Test/SerializeTest.cs b/dotnet/Serpent.Test/SerializeTest.cs index f0cf427..d7c4850 100644 --- a/dotnet/Serpent.Test/SerializeTest.cs +++ b/dotnet/Serpent.Test/SerializeTest.cs @@ -318,6 +318,26 @@ public void TestBytes() Parser p = new Parser(); string parsed = p.Parse(ser).Root.ToString(); Assert.AreEqual(39, parsed.Length); + + IDictionary dict = new Dictionary { + {"data", "YWJjZGVm"}, + {"encoding", "base64"} + }; + byte[] bytes2 = Serializer.ToBytes(dict); + Assert.AreEqual(bytes, bytes2); + + dict["encoding"] = "base99"; + Assert.Throws(()=>Serializer.ToBytes(dict)); + dict.Clear(); + Assert.Throws(()=>Serializer.ToBytes(dict)); + dict.Clear(); + dict["data"] = "YWJjZGVm"; + Assert.Throws(()=>Serializer.ToBytes(dict)); + dict.Clear(); + dict["encoding"] = "base64"; + Assert.Throws(()=>Serializer.ToBytes(dict)); + Assert.Throws(()=>Serializer.ToBytes(12345)); + Assert.Throws(()=>Serializer.ToBytes(null)); } [Test] diff --git a/dotnet/Serpent/Parser.cs b/dotnet/Serpent/Parser.cs index 33ab0d2..8a59191 100644 --- a/dotnet/Serpent/Parser.cs +++ b/dotnet/Serpent/Parser.cs @@ -506,7 +506,7 @@ double ParseImaginaryPart(SeekableStringReader sr) char j_char = sr.Read(); if(j_char!='j') throw new ParseException("not an imaginary part"); - } catch (IndexOutOfRangeException x) { + } catch (IndexOutOfRangeException) { throw new ParseException("not an imaginary part"); } return double_value; diff --git a/dotnet/Serpent/Serializer.cs b/dotnet/Serpent/Serializer.cs index f2a7cee..956274d 100644 --- a/dotnet/Serpent/Serializer.cs +++ b/dotnet/Serpent/Serializer.cs @@ -503,5 +503,33 @@ protected Func GetCustomConverter(Type obj_type) return null; } + + /// + /// Utility function to convert obj back to actual bytes if it is a serpent-encoded bytes dictionary + /// (a IDictionary with base-64 encoded 'data' in it and 'encoding'='base64'). + /// If obj is already a byte array, return obj unmodified. + /// If it is something else, throw an ArgumentException + /// + public static byte[] ToBytes(object obj) { + IDictionary dict = obj as IDictionary; + if(dict!=null) + { + string data; + string encoding; + bool hasData = dict.TryGetValue("data", out data); + bool hasEncoding = dict.TryGetValue("encoding", out encoding); + if(!hasData || !hasEncoding || encoding!="base64") + { + throw new ArgumentException("argument is neither bytearray nor serpent base64 encoded bytes dict"); + } + return Convert.FromBase64String(data); + } + byte[] bytearray = obj as byte[]; + if(bytearray!=null) + { + return bytearray; + } + throw new ArgumentException("argument is neither bytearray nor serpent base64 encoded bytes dict"); + } } } diff --git a/dotnet/Serpent/Serpent.nuspec b/dotnet/Serpent/Serpent.nuspec index d4a8092..d83da19 100644 --- a/dotnet/Serpent/Serpent.nuspec +++ b/dotnet/Serpent/Serpent.nuspec @@ -18,7 +18,10 @@ such a Python expression itself, back into the equivalent .NET datatypes. More info for the Python version is on Pypi: https://pypi.python.org/pypi/serpent Serpent Python literal expression serialization - custom converters now work for inheritance tree rather than just one specific class + +Custom converters now work for inheritance tree rather than just one specific class. +Added Serializer.ToBytes utility method to decode serpent base-64 encoded byte arrays. + Copyright 2016 serialization python pyro From 3b00afc0abdf3447c2f6d531aae307082ebf5c31 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Mon, 10 Oct 2016 23:40:15 +0200 Subject: [PATCH 021/230] java added Serializer.toBytes utility function to decode a serpent base-64 encoded byte array back into the actual byte array --- .../net/razorvine/serpent/Serializer.java | 25 +++++++++ .../razorvine/serpent/test/SerializeTest.java | 51 +++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/java/src/main/java/net/razorvine/serpent/Serializer.java b/java/src/main/java/net/razorvine/serpent/Serializer.java index 21bc0b4..a840d39 100644 --- a/java/src/main/java/net/razorvine/serpent/Serializer.java +++ b/java/src/main/java/net/razorvine/serpent/Serializer.java @@ -543,4 +543,29 @@ protected void serialize_exception(Exception ex, PrintWriter p, int level) } serialize_dict(dict, p, level); } + + /** + * Utility function to convert obj back to actual bytes if it is a serpent-encoded bytes dictionary + * (a IDictionary with base-64 encoded 'data' in it and 'encoding'='base64'). + * If obj is already a byte array, return obj unmodified. + * If it is something else, throw an IllegalArgumentException + */ + public static byte[] toBytes(Object obj) { + if(obj instanceof Map) + { + @SuppressWarnings("unchecked") + Map dict = (Map)obj; + String data = dict.get("data"); + String encoding = dict.get("encoding"); + if(data==null || encoding==null || !encoding.equals("base64")) + { + throw new IllegalArgumentException("argument is neither bytearray nor serpent base64 encoded bytes dict"); + } + return DatatypeConverter.parseBase64Binary(data); + } + if(obj instanceof byte[]) + { + return (byte[]) obj; + } + throw new IllegalArgumentException("argument is neither bytearray nor serpent base64 encoded bytes dict"); } } diff --git a/java/src/test/java/net/razorvine/serpent/test/SerializeTest.java b/java/src/test/java/net/razorvine/serpent/test/SerializeTest.java index f9e500d..971c4c0 100644 --- a/java/src/test/java/net/razorvine/serpent/test/SerializeTest.java +++ b/java/src/test/java/net/razorvine/serpent/test/SerializeTest.java @@ -193,6 +193,57 @@ public void testBytes() Parser p = new Parser(); String parsed = p.parse(ser).root.toString(); assertEquals(39, parsed.length()); + + Map dict = new HashMap(); + dict.put("data", "YWJjZGVm"); + dict.put("encoding", "base64"); + + byte[] bytes2 = Serializer.toBytes(dict); + assertArrayEquals(bytes, bytes2); + + dict.put("encoding", "base99"); + try { + Serializer.toBytes(dict); + fail("error expected"); + } catch (IllegalArgumentException x) { + // + } + + dict.clear(); + try { + Serializer.toBytes(dict); + fail("error expected"); + } catch (IllegalArgumentException x) { + // + } + dict.clear(); + dict.put("data", "YWJjZGVm"); + try { + Serializer.toBytes(dict); + fail("error expected"); + } catch (IllegalArgumentException x) { + // + } + dict.clear(); + dict.put("encoding", "base64"); + try { + Serializer.toBytes(dict); + fail("error expected"); + } catch (IllegalArgumentException x) { + // + } + try { + Serializer.toBytes(12345); + fail("error expected"); + } catch (IllegalArgumentException x) { + // + } + try { + Serializer.toBytes(null); + fail("error expected"); + } catch (IllegalArgumentException x) { + // + } } From 11a80c1c4fdd0199be30a6a8d1f28263ec4bec75 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Mon, 10 Oct 2016 23:46:16 +0200 Subject: [PATCH 022/230] bump to version 1.15 --- dotnet/Serpent/Properties/AssemblyInfo.cs | 4 ++-- dotnet/Serpent/Serpent.nuspec | 2 +- dotnet/Serpent/Version.cs | 2 +- java/pom.xml | 2 +- java/src/main/java/net/razorvine/serpent/LibraryVersion.java | 2 +- java/src/main/java/net/razorvine/serpent/package-info.java | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dotnet/Serpent/Properties/AssemblyInfo.cs b/dotnet/Serpent/Properties/AssemblyInfo.cs index d58e8ce..66268ca 100644 --- a/dotnet/Serpent/Properties/AssemblyInfo.cs +++ b/dotnet/Serpent/Properties/AssemblyInfo.cs @@ -36,5 +36,5 @@ decoded there (using ast.literal_eval()). It can ofcourse also deserialize // // You can specify all the values or you can use the default the Revision and // Build Numbers by using the '*' as shown below: -[assembly: AssemblyVersion("1.13.0.*")] -[assembly: AssemblyFileVersion("1.13.0.0")] +[assembly: AssemblyVersion("1.15.0.*")] +[assembly: AssemblyFileVersion("1.15.0.0")] diff --git a/dotnet/Serpent/Serpent.nuspec b/dotnet/Serpent/Serpent.nuspec index d83da19..e23bb49 100644 --- a/dotnet/Serpent/Serpent.nuspec +++ b/dotnet/Serpent/Serpent.nuspec @@ -2,7 +2,7 @@ Razorvine.Serpent - 1.13.0.0 + 1.15.0.0 Razorvine.Serpent Irmen de Jong Irmen de Jong diff --git a/dotnet/Serpent/Version.cs b/dotnet/Serpent/Version.cs index 3fa80d9..3314e82 100644 --- a/dotnet/Serpent/Version.cs +++ b/dotnet/Serpent/Version.cs @@ -10,6 +10,6 @@ namespace Razorvine.Serpent { public static class LibraryVersion { - public static string Version = "1.13"; + public static string Version = "1.15"; } } diff --git a/java/pom.xml b/java/pom.xml index 0a9d9bb..f101ad0 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -9,7 +9,7 @@ net.razorvine serpent - 1.14-SNAPSHOT + 1.15-SNAPSHOT jar serpent diff --git a/java/src/main/java/net/razorvine/serpent/LibraryVersion.java b/java/src/main/java/net/razorvine/serpent/LibraryVersion.java index 1ca1064..b7a0159 100644 --- a/java/src/main/java/net/razorvine/serpent/LibraryVersion.java +++ b/java/src/main/java/net/razorvine/serpent/LibraryVersion.java @@ -8,5 +8,5 @@ package net.razorvine.serpent; public final class LibraryVersion { - public static final String VERSION = "1.13"; + public static final String VERSION = "1.15"; } diff --git a/java/src/main/java/net/razorvine/serpent/package-info.java b/java/src/main/java/net/razorvine/serpent/package-info.java index 7852286..a473a31 100644 --- a/java/src/main/java/net/razorvine/serpent/package-info.java +++ b/java/src/main/java/net/razorvine/serpent/package-info.java @@ -3,7 +3,7 @@ * (a.k.a. Python's ast.literal_eval in Java) * Software license: "MIT software license". See http://opensource.org/licenses/MIT * @author Irmen de Jong (irmen@razorvine.net) - * @version 1.13 + * @version 1.15 */ package net.razorvine.serpent; From ad04a1cc510697ed158f6e12c5e3030f527ef33e Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 11 Oct 2016 00:05:29 +0200 Subject: [PATCH 023/230] dropped support for python 2.6 and python 3.2 --- .travis.yml | 4 ++-- README.md | 5 +---- serpent.py | 7 ++----- setup.py | 9 ++++++--- test_serpent.py | 2 +- tox.ini | 2 +- 6 files changed, 13 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 35648d6..ba7ec44 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,11 @@ language: python python: - - "pypy" - "2.7" - - "pypy3" - "3.3" - "3.4" - "3.5" + - "3.6" + - "pypy" # Use fast travis build infrastructure explicitly sudo: false diff --git a/README.md b/README.md index bb6f31b..5fae2ce 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Example usage can be found in ./java/test/SerpentExample.java SOME MORE DETAILS ----------------- -Compatible with Python 2.6+ (including 3.x), IronPython 2.7+, Jython 2.7+. +Compatible with Python 2.7+ (including 3.x), IronPython 2.7+, Jython 2.7+. Serpent handles several special Python types to make life easier: @@ -65,9 +65,6 @@ The serializer is not thread-safe. Make sure you're not making changes to the object tree that is being serialized, and don't use the same serializer in different threads. -Python 2.6 cannot deserialize complex numbers at all (limitation of -``ast.literal_eval`` in 2.6). - Because the serialized format is just valid Python source code, it can contain comments. Serpent does not add comments by itself apart from the single header line. diff --git a/serpent.py b/serpent.py index 4af577d..b1a38f6 100644 --- a/serpent.py +++ b/serpent.py @@ -7,7 +7,7 @@ machines over the network for instance (because only 'safe' literals are encoded). -Compatible with Python 2.6+ (including 3.x), IronPython 2.7+, Jython 2.7+. +Compatible with Python 2.7+ (including 3.x), IronPython 2.7+, Jython 2.7+. Serpent handles several special Python types to make life easier: @@ -32,9 +32,6 @@ to the object tree that is being serialized, and don't use the same serializer in different threads. -Python 2.6 cannot deserialize complex numbers (limitation of -ast.literal_eval in 2.6) - Because the serialized format is just valid Python source code, it can contain comments. @@ -270,7 +267,7 @@ def serialize(self, obj): if self.set_literals: header += "python3.2\n" # set-literals require python 3.2+ to deserialize (ast.literal_eval limitation) else: - header += "python2.6\n" + header += "python2.6\n" # don't change this even though we don't support 2.6 any longer, otherwise we can't read older serpent strings out = [header.encode("utf-8")] try: if os.name != "java" and sys.platform != "cli": diff --git a/setup.py b/setup.py index 0f67b08..be962fd 100755 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ Serpent allows comments in the serialized data (because it is just Python source code). Serpent can't serialize object graphs (when an object refers to itself); it will then crash with a ValueError pointing out the problem. -Works with Python 2.6+ (including 3.x), IronPython 2.7+, Jython 2.7+. +Works with Python 2.7+ (including 3.x), IronPython 2.7+, Jython 2.7+. **FAQ** @@ -162,8 +162,11 @@ def __init__(self): "Natural Language :: English", "Operating System :: OS Independent", "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 3", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", "Topic :: Software Development" ], diff --git a/test_serpent.py b/test_serpent.py index 478c433..16a0b70 100644 --- a/test_serpent.py +++ b/test_serpent.py @@ -97,7 +97,7 @@ def test_header(self): else: self.assertTrue(type(ser) is bytes) header, _, rest = ser.partition(b"\n") - hdr = "# serpent utf-8 python2.6".encode("utf-8") + hdr = "# serpent utf-8 python2.6".encode("utf-8") # don't change the 2.6 here even though we don't support python 2.6 any longer self.assertEqual(hdr, header) def test_comments(self): diff --git a/tox.ini b/tox.ini index 8168830..e3bc339 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=py26,py27,py32,py33,py34,py35,py36,pypy +envlist=py27,py33,py34,py35,py36,pypy [testenv] commands=python -bb -E test_serpent.py From af8bee351f2da43d70579586600fcfa85feec39a Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 11 Oct 2016 00:07:18 +0200 Subject: [PATCH 024/230] travis doesn't have python 3.6 yet --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ba7ec44..aead5b2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,6 @@ python: - "3.3" - "3.4" - "3.5" - - "3.6" - "pypy" # Use fast travis build infrastructure explicitly From 6995d0dad360f5c7eac814e6553bfc9cfc167758 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 11 Oct 2016 00:43:13 +0200 Subject: [PATCH 025/230] Serializer.toBytes moved to Parser.toBytes --- dotnet/Serpent.Test/SerializeTest.cs | 14 ++++----- dotnet/Serpent/Parser.cs | 29 +++++++++++++++++++ dotnet/Serpent/Serializer.cs | 28 ------------------ .../java/net/razorvine/serpent/Parser.java | 28 ++++++++++++++++++ .../net/razorvine/serpent/Serializer.java | 25 ---------------- .../razorvine/serpent/test/SerializeTest.java | 14 ++++----- 6 files changed, 71 insertions(+), 67 deletions(-) diff --git a/dotnet/Serpent.Test/SerializeTest.cs b/dotnet/Serpent.Test/SerializeTest.cs index d7c4850..4286de3 100644 --- a/dotnet/Serpent.Test/SerializeTest.cs +++ b/dotnet/Serpent.Test/SerializeTest.cs @@ -323,21 +323,21 @@ public void TestBytes() {"data", "YWJjZGVm"}, {"encoding", "base64"} }; - byte[] bytes2 = Serializer.ToBytes(dict); + byte[] bytes2 = Parser.ToBytes(dict); Assert.AreEqual(bytes, bytes2); dict["encoding"] = "base99"; - Assert.Throws(()=>Serializer.ToBytes(dict)); + Assert.Throws(()=>Parser.ToBytes(dict)); dict.Clear(); - Assert.Throws(()=>Serializer.ToBytes(dict)); + Assert.Throws(()=>Parser.ToBytes(dict)); dict.Clear(); dict["data"] = "YWJjZGVm"; - Assert.Throws(()=>Serializer.ToBytes(dict)); + Assert.Throws(()=>Parser.ToBytes(dict)); dict.Clear(); dict["encoding"] = "base64"; - Assert.Throws(()=>Serializer.ToBytes(dict)); - Assert.Throws(()=>Serializer.ToBytes(12345)); - Assert.Throws(()=>Serializer.ToBytes(null)); + Assert.Throws(()=>Parser.ToBytes(dict)); + Assert.Throws(()=>Parser.ToBytes(12345)); + Assert.Throws(()=>Parser.ToBytes(null)); } [Test] diff --git a/dotnet/Serpent/Parser.cs b/dotnet/Serpent/Parser.cs index 8a59191..7e9edb1 100644 --- a/dotnet/Serpent/Parser.cs +++ b/dotnet/Serpent/Parser.cs @@ -601,6 +601,35 @@ double ParseDouble(string numberstr) if(numberstr=="-1e30000") return double.NegativeInfinity; return double.Parse(numberstr, CultureInfo.InvariantCulture); } + + + /// + /// Utility function to convert obj back to actual bytes if it is a serpent-encoded bytes dictionary + /// (a IDictionary with base-64 encoded 'data' in it and 'encoding'='base64'). + /// If obj is already a byte array, return obj unmodified. + /// If it is something else, throw an ArgumentException + /// + public static byte[] ToBytes(object obj) { + IDictionary dict = obj as IDictionary; + if(dict!=null) + { + string data; + string encoding; + bool hasData = dict.TryGetValue("data", out data); + bool hasEncoding = dict.TryGetValue("encoding", out encoding); + if(!hasData || !hasEncoding || encoding!="base64") + { + throw new ArgumentException("argument is neither bytearray nor serpent base64 encoded bytes dict"); + } + return Convert.FromBase64String(data); + } + byte[] bytearray = obj as byte[]; + if(bytearray!=null) + { + return bytearray; + } + throw new ArgumentException("argument is neither bytearray nor serpent base64 encoded bytes dict"); + } } } diff --git a/dotnet/Serpent/Serializer.cs b/dotnet/Serpent/Serializer.cs index 956274d..f2a7cee 100644 --- a/dotnet/Serpent/Serializer.cs +++ b/dotnet/Serpent/Serializer.cs @@ -503,33 +503,5 @@ protected Func GetCustomConverter(Type obj_type) return null; } - - /// - /// Utility function to convert obj back to actual bytes if it is a serpent-encoded bytes dictionary - /// (a IDictionary with base-64 encoded 'data' in it and 'encoding'='base64'). - /// If obj is already a byte array, return obj unmodified. - /// If it is something else, throw an ArgumentException - /// - public static byte[] ToBytes(object obj) { - IDictionary dict = obj as IDictionary; - if(dict!=null) - { - string data; - string encoding; - bool hasData = dict.TryGetValue("data", out data); - bool hasEncoding = dict.TryGetValue("encoding", out encoding); - if(!hasData || !hasEncoding || encoding!="base64") - { - throw new ArgumentException("argument is neither bytearray nor serpent base64 encoded bytes dict"); - } - return Convert.FromBase64String(data); - } - byte[] bytearray = obj as byte[]; - if(bytearray!=null) - { - return bytearray; - } - throw new ArgumentException("argument is neither bytearray nor serpent base64 encoded bytes dict"); - } } } diff --git a/java/src/main/java/net/razorvine/serpent/Parser.java b/java/src/main/java/net/razorvine/serpent/Parser.java index 67b53d0..0783e86 100644 --- a/java/src/main/java/net/razorvine/serpent/Parser.java +++ b/java/src/main/java/net/razorvine/serpent/Parser.java @@ -16,6 +16,8 @@ import java.util.Map; import java.util.Set; +import javax.xml.bind.DatatypeConverter; + import net.razorvine.serpent.ast.*; @@ -581,4 +583,30 @@ NoneNode parseNone(SeekableStringReader sr) return NoneNode.Instance; throw new ParseException("expected None"); } + + /** + * Utility function to convert obj back to actual bytes if it is a serpent-encoded bytes dictionary + * (a IDictionary with base-64 encoded 'data' in it and 'encoding'='base64'). + * If obj is already a byte array, return obj unmodified. + * If it is something else, throw an IllegalArgumentException + */ + public static byte[] toBytes(Object obj) { + if(obj instanceof Map) + { + @SuppressWarnings("unchecked") + Map dict = (Map)obj; + String data = dict.get("data"); + String encoding = dict.get("encoding"); + if(data==null || encoding==null || !encoding.equals("base64")) + { + throw new IllegalArgumentException("argument is neither bytearray nor serpent base64 encoded bytes dict"); + } + return DatatypeConverter.parseBase64Binary(data); + } + if(obj instanceof byte[]) + { + return (byte[]) obj; + } + throw new IllegalArgumentException("argument is neither bytearray nor serpent base64 encoded bytes dict"); + } } diff --git a/java/src/main/java/net/razorvine/serpent/Serializer.java b/java/src/main/java/net/razorvine/serpent/Serializer.java index a840d39..21bc0b4 100644 --- a/java/src/main/java/net/razorvine/serpent/Serializer.java +++ b/java/src/main/java/net/razorvine/serpent/Serializer.java @@ -543,29 +543,4 @@ protected void serialize_exception(Exception ex, PrintWriter p, int level) } serialize_dict(dict, p, level); } - - /** - * Utility function to convert obj back to actual bytes if it is a serpent-encoded bytes dictionary - * (a IDictionary with base-64 encoded 'data' in it and 'encoding'='base64'). - * If obj is already a byte array, return obj unmodified. - * If it is something else, throw an IllegalArgumentException - */ - public static byte[] toBytes(Object obj) { - if(obj instanceof Map) - { - @SuppressWarnings("unchecked") - Map dict = (Map)obj; - String data = dict.get("data"); - String encoding = dict.get("encoding"); - if(data==null || encoding==null || !encoding.equals("base64")) - { - throw new IllegalArgumentException("argument is neither bytearray nor serpent base64 encoded bytes dict"); - } - return DatatypeConverter.parseBase64Binary(data); - } - if(obj instanceof byte[]) - { - return (byte[]) obj; - } - throw new IllegalArgumentException("argument is neither bytearray nor serpent base64 encoded bytes dict"); } } diff --git a/java/src/test/java/net/razorvine/serpent/test/SerializeTest.java b/java/src/test/java/net/razorvine/serpent/test/SerializeTest.java index 971c4c0..3455ff7 100644 --- a/java/src/test/java/net/razorvine/serpent/test/SerializeTest.java +++ b/java/src/test/java/net/razorvine/serpent/test/SerializeTest.java @@ -198,12 +198,12 @@ public void testBytes() dict.put("data", "YWJjZGVm"); dict.put("encoding", "base64"); - byte[] bytes2 = Serializer.toBytes(dict); + byte[] bytes2 = Parser.toBytes(dict); assertArrayEquals(bytes, bytes2); dict.put("encoding", "base99"); try { - Serializer.toBytes(dict); + Parser.toBytes(dict); fail("error expected"); } catch (IllegalArgumentException x) { // @@ -211,7 +211,7 @@ public void testBytes() dict.clear(); try { - Serializer.toBytes(dict); + Parser.toBytes(dict); fail("error expected"); } catch (IllegalArgumentException x) { // @@ -219,7 +219,7 @@ public void testBytes() dict.clear(); dict.put("data", "YWJjZGVm"); try { - Serializer.toBytes(dict); + Parser.toBytes(dict); fail("error expected"); } catch (IllegalArgumentException x) { // @@ -227,19 +227,19 @@ public void testBytes() dict.clear(); dict.put("encoding", "base64"); try { - Serializer.toBytes(dict); + Parser.toBytes(dict); fail("error expected"); } catch (IllegalArgumentException x) { // } try { - Serializer.toBytes(12345); + Parser.toBytes(12345); fail("error expected"); } catch (IllegalArgumentException x) { // } try { - Serializer.toBytes(null); + Parser.toBytes(null); fail("error expected"); } catch (IllegalArgumentException x) { // From bf394e85d7ecd23e3e1d641e5e78c5b392be1011 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 11 Oct 2016 00:59:54 +0200 Subject: [PATCH 026/230] removed old copyright year, fixed nuspec releasenote --- dotnet/Serpent.Test/CycleTest.cs | 2 +- dotnet/Serpent.Test/ParserTest.cs | 2 +- .../Serpent.Test/Properties/AssemblyInfo.cs | 62 ++-- dotnet/Serpent.Test/SerializeTest.cs | 2 +- dotnet/Serpent.Test/StringreaderTest.cs | 2 +- dotnet/Serpent/Ast.cs | 2 +- dotnet/Serpent/ComplexNumber.cs | 178 +++++------ dotnet/Serpent/DebugVisitor.cs | 286 ++++++++--------- dotnet/Serpent/ObjectifyVisitor.cs | 298 +++++++++--------- dotnet/Serpent/ParseException.cs | 58 ++-- dotnet/Serpent/Parser.cs | 2 +- dotnet/Serpent/SeekableStringReader.cs | 2 +- dotnet/Serpent/Serializer.cs | 2 +- dotnet/Serpent/Serpent.nuspec | 2 +- 14 files changed, 450 insertions(+), 450 deletions(-) diff --git a/dotnet/Serpent.Test/CycleTest.cs b/dotnet/Serpent.Test/CycleTest.cs index c18dc0b..5096156 100644 --- a/dotnet/Serpent.Test/CycleTest.cs +++ b/dotnet/Serpent.Test/CycleTest.cs @@ -2,7 +2,7 @@ /// Serpent, a Python literal expression serializer/deserializer /// (a.k.a. Python's ast.literal_eval in .NET) /// -/// Copyright 2014, Irmen de Jong (irmen@razorvine.net) +/// Copyright Irmen de Jong (irmen@razorvine.net) /// Software license: "MIT software license". See http://opensource.org/licenses/MIT /// diff --git a/dotnet/Serpent.Test/ParserTest.cs b/dotnet/Serpent.Test/ParserTest.cs index 175cb57..41ef90a 100644 --- a/dotnet/Serpent.Test/ParserTest.cs +++ b/dotnet/Serpent.Test/ParserTest.cs @@ -2,7 +2,7 @@ /// Serpent, a Python literal expression serializer/deserializer /// (a.k.a. Python's ast.literal_eval in .NET) /// -/// Copyright 2014, Irmen de Jong (irmen@razorvine.net) +/// Copyright Irmen de Jong (irmen@razorvine.net) /// Software license: "MIT software license". See http://opensource.org/licenses/MIT /// diff --git a/dotnet/Serpent.Test/Properties/AssemblyInfo.cs b/dotnet/Serpent.Test/Properties/AssemblyInfo.cs index 004369b..2e6d9da 100644 --- a/dotnet/Serpent.Test/Properties/AssemblyInfo.cs +++ b/dotnet/Serpent.Test/Properties/AssemblyInfo.cs @@ -1,31 +1,31 @@ -#region Using directives - -using System; -using System.Reflection; -using System.Runtime.InteropServices; - -#endregion - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("Serpent.Test")] -[assembly: AssemblyDescription("")] -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("")] -[assembly: AssemblyProduct("Serpent.Test")] -[assembly: AssemblyCopyright("Copyright 2014")] -[assembly: AssemblyTrademark("")] -[assembly: AssemblyCulture("")] - -// This sets the default COM visibility of types in the assembly to invisible. -// If you need to expose a type to COM, use [ComVisible(true)] on that type. -[assembly: ComVisible(false)] - -// The assembly version has following format : -// -// Major.Minor.Build.Revision -// -// You can specify all the values or you can use the default the Revision and -// Build Numbers by using the '*' as shown below: -[assembly: AssemblyVersion("1.0.*")] +#region Using directives + +using System; +using System.Reflection; +using System.Runtime.InteropServices; + +#endregion + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Serpent.Test")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Serpent.Test")] +[assembly: AssemblyCopyright("Copyright Irmen de Jong")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// This sets the default COM visibility of types in the assembly to invisible. +// If you need to expose a type to COM, use [ComVisible(true)] on that type. +[assembly: ComVisible(false)] + +// The assembly version has following format : +// +// Major.Minor.Build.Revision +// +// You can specify all the values or you can use the default the Revision and +// Build Numbers by using the '*' as shown below: +[assembly: AssemblyVersion("1.0.*")] diff --git a/dotnet/Serpent.Test/SerializeTest.cs b/dotnet/Serpent.Test/SerializeTest.cs index 4286de3..6ef1cba 100644 --- a/dotnet/Serpent.Test/SerializeTest.cs +++ b/dotnet/Serpent.Test/SerializeTest.cs @@ -2,7 +2,7 @@ /// Serpent, a Python literal expression serializer/deserializer /// (a.k.a. Python's ast.literal_eval in .NET) /// -/// Copyright 2014, Irmen de Jong (irmen@razorvine.net) +/// Copyright Irmen de Jong (irmen@razorvine.net) /// Software license: "MIT software license". See http://opensource.org/licenses/MIT /// diff --git a/dotnet/Serpent.Test/StringreaderTest.cs b/dotnet/Serpent.Test/StringreaderTest.cs index 0494925..7a5f94c 100644 --- a/dotnet/Serpent.Test/StringreaderTest.cs +++ b/dotnet/Serpent.Test/StringreaderTest.cs @@ -2,7 +2,7 @@ /// Serpent, a Python literal expression serializer/deserializer /// (a.k.a. Python's ast.literal_eval in .NET) /// -/// Copyright 2014, Irmen de Jong (irmen@razorvine.net) +/// Copyright Irmen de Jong (irmen@razorvine.net) /// Software license: "MIT software license". See http://opensource.org/licenses/MIT /// diff --git a/dotnet/Serpent/Ast.cs b/dotnet/Serpent/Ast.cs index 9eede1b..932c444 100644 --- a/dotnet/Serpent/Ast.cs +++ b/dotnet/Serpent/Ast.cs @@ -2,7 +2,7 @@ /// Serpent, a Python literal expression serializer/deserializer /// (a.k.a. Python's ast.literal_eval in .NET) /// -/// Copyright 2014, Irmen de Jong (irmen@razorvine.net) +/// Copyright Irmen de Jong (irmen@razorvine.net) /// Software license: "MIT software license". See http://opensource.org/licenses/MIT /// diff --git a/dotnet/Serpent/ComplexNumber.cs b/dotnet/Serpent/ComplexNumber.cs index 286c1f1..c617bf7 100644 --- a/dotnet/Serpent/ComplexNumber.cs +++ b/dotnet/Serpent/ComplexNumber.cs @@ -1,90 +1,90 @@ -/// -/// Serpent, a Python literal expression serializer/deserializer -/// (a.k.a. Python's ast.literal_eval in .NET) -/// -/// Copyright 2014, Irmen de Jong (irmen@razorvine.net) -/// Software license: "MIT software license". See http://opensource.org/licenses/MIT -/// - -using System; -using System.Text; - -namespace Razorvine.Serpent -{ - -/// -/// A Complex Number class. -/// -[Serializable] -public class ComplexNumber { - - public double Real {get; private set;} - public double Imaginary {get; private set;} - - public ComplexNumber(double r, double i) { - Real=r; - Imaginary=i; - } - - public override string ToString() - { - StringBuilder sb=new StringBuilder(); - sb.Append(Real); - if(Imaginary>0) - sb.Append('+'); - return sb.Append(Imaginary).Append('i').ToString(); - } - - public double Magnitude() { - return Math.Sqrt(Real * Real + Imaginary * Imaginary); - } - - public static ComplexNumber operator +(ComplexNumber c1, ComplexNumber c2) { - return new ComplexNumber(c1.Real + c2.Real, c1.Imaginary + c2.Imaginary); - } - - - public static ComplexNumber operator -(ComplexNumber c1, ComplexNumber c2) { - return new ComplexNumber(c1.Real - c2.Real, c1.Imaginary - c2.Imaginary); - } - - public static ComplexNumber operator *(ComplexNumber c1, ComplexNumber c2) { - return new ComplexNumber(c1.Real * c2.Real - c1.Imaginary * c2.Imaginary, c1.Real * c2.Imaginary + c1.Imaginary * c2.Real); - } - - public static ComplexNumber operator /(ComplexNumber c1, ComplexNumber c2) { - return new ComplexNumber((c1.Real * c2.Real + c1.Imaginary * c2.Imaginary) / (c2.Real * c2.Real + c2.Imaginary * c2.Imaginary), (c1.Imaginary * c2.Real - c1.Real * c2.Imaginary) - / (c2.Real * c2.Real + c2.Imaginary * c2.Imaginary)); - } - - #region Equals and GetHashCode implementation - public override bool Equals(object obj) - { - if(!(obj is ComplexNumber)) - return false; - ComplexNumber other = (ComplexNumber) obj; - return Real==other.Real && Imaginary==other.Imaginary; - } - - public override int GetHashCode() - { - return (Real.GetHashCode()) ^ (Imaginary.GetHashCode()); - } - - public static bool operator ==(ComplexNumber lhs, ComplexNumber rhs) - { - if (ReferenceEquals(lhs, rhs)) - return true; - if (ReferenceEquals(lhs, null) || ReferenceEquals(rhs, null)) - return false; - return lhs.Equals(rhs); - } - - public static bool operator !=(ComplexNumber lhs, ComplexNumber rhs) - { - return !(lhs == rhs); - } - #endregion -} - +/// +/// Serpent, a Python literal expression serializer/deserializer +/// (a.k.a. Python's ast.literal_eval in .NET) +/// +/// Copyright Irmen de Jong (irmen@razorvine.net) +/// Software license: "MIT software license". See http://opensource.org/licenses/MIT +/// + +using System; +using System.Text; + +namespace Razorvine.Serpent +{ + +/// +/// A Complex Number class. +/// +[Serializable] +public class ComplexNumber { + + public double Real {get; private set;} + public double Imaginary {get; private set;} + + public ComplexNumber(double r, double i) { + Real=r; + Imaginary=i; + } + + public override string ToString() + { + StringBuilder sb=new StringBuilder(); + sb.Append(Real); + if(Imaginary>0) + sb.Append('+'); + return sb.Append(Imaginary).Append('i').ToString(); + } + + public double Magnitude() { + return Math.Sqrt(Real * Real + Imaginary * Imaginary); + } + + public static ComplexNumber operator +(ComplexNumber c1, ComplexNumber c2) { + return new ComplexNumber(c1.Real + c2.Real, c1.Imaginary + c2.Imaginary); + } + + + public static ComplexNumber operator -(ComplexNumber c1, ComplexNumber c2) { + return new ComplexNumber(c1.Real - c2.Real, c1.Imaginary - c2.Imaginary); + } + + public static ComplexNumber operator *(ComplexNumber c1, ComplexNumber c2) { + return new ComplexNumber(c1.Real * c2.Real - c1.Imaginary * c2.Imaginary, c1.Real * c2.Imaginary + c1.Imaginary * c2.Real); + } + + public static ComplexNumber operator /(ComplexNumber c1, ComplexNumber c2) { + return new ComplexNumber((c1.Real * c2.Real + c1.Imaginary * c2.Imaginary) / (c2.Real * c2.Real + c2.Imaginary * c2.Imaginary), (c1.Imaginary * c2.Real - c1.Real * c2.Imaginary) + / (c2.Real * c2.Real + c2.Imaginary * c2.Imaginary)); + } + + #region Equals and GetHashCode implementation + public override bool Equals(object obj) + { + if(!(obj is ComplexNumber)) + return false; + ComplexNumber other = (ComplexNumber) obj; + return Real==other.Real && Imaginary==other.Imaginary; + } + + public override int GetHashCode() + { + return (Real.GetHashCode()) ^ (Imaginary.GetHashCode()); + } + + public static bool operator ==(ComplexNumber lhs, ComplexNumber rhs) + { + if (ReferenceEquals(lhs, rhs)) + return true; + if (ReferenceEquals(lhs, null) || ReferenceEquals(rhs, null)) + return false; + return lhs.Equals(rhs); + } + + public static bool operator !=(ComplexNumber lhs, ComplexNumber rhs) + { + return !(lhs == rhs); + } + #endregion +} + } \ No newline at end of file diff --git a/dotnet/Serpent/DebugVisitor.cs b/dotnet/Serpent/DebugVisitor.cs index 485411e..1d759b3 100644 --- a/dotnet/Serpent/DebugVisitor.cs +++ b/dotnet/Serpent/DebugVisitor.cs @@ -1,143 +1,143 @@ -/// -/// Serpent, a Python literal expression serializer/deserializer -/// (a.k.a. Python's ast.literal_eval in .NET) -/// -/// Copyright 2014, Irmen de Jong (irmen@razorvine.net) -/// Software license: "MIT software license". See http://opensource.org/licenses/MIT -/// - -using System; -using System.Collections.Generic; -using System.Text; - -namespace Razorvine.Serpent -{ - /// - /// Ast nodevisitor that prints out the Ast as a string for debugging purposes - /// - public class DebugVisitor: Ast.INodeVisitor - { - private StringBuilder result = new StringBuilder(); - private int indent=0; - - public DebugVisitor() - { - } - - /// - /// Get the debug string representation result. - /// - public override string ToString() - { - return result.ToString(); - } - - protected void Indent() - { - for(int i=0; i +/// Serpent, a Python literal expression serializer/deserializer +/// (a.k.a. Python's ast.literal_eval in .NET) +/// +/// Copyright Irmen de Jong (irmen@razorvine.net) +/// Software license: "MIT software license". See http://opensource.org/licenses/MIT +/// + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Razorvine.Serpent +{ + /// + /// Ast nodevisitor that prints out the Ast as a string for debugging purposes + /// + public class DebugVisitor: Ast.INodeVisitor + { + private StringBuilder result = new StringBuilder(); + private int indent=0; + + public DebugVisitor() + { + } + + /// + /// Get the debug string representation result. + /// + public override string ToString() + { + return result.ToString(); + } + + protected void Indent() + { + for(int i=0; i -/// Serpent, a Python literal expression serializer/deserializer -/// (a.k.a. Python's ast.literal_eval in .NET) -/// -/// Copyright 2014, Irmen de Jong (irmen@razorvine.net) -/// Software license: "MIT software license". See http://opensource.org/licenses/MIT -/// - -using System; -using System.Collections.Generic; -using System.Collections; - -namespace Razorvine.Serpent -{ - /// - /// Ast nodevisitor that turns the AST into actual .NET objects (array, int, IDictionary, string, etc...) - /// - public class ObjectifyVisitor: Ast.INodeVisitor - { - private Stack generated = new Stack(); - private Func dictToInstance = null; - - /// - /// Create the visitor that converts AST in actual objects. - /// - public ObjectifyVisitor() - { - } - - /// - /// Create the visitor that converts AST in actual objects. - /// - /// functin to convert dicts to actual instances for a class, - /// instead of leaving them as dictionaries. Requires the __class__ key to be present - /// in the dict node. If it returns null, the normal processing is done. - public ObjectifyVisitor(Func dictToInstance) - { - this.dictToInstance = dictToInstance; - } - - /// - /// get the resulting object tree. - /// - public object GetObject() - { - return generated.Pop(); - } - - public void Visit(Ast.ComplexNumberNode complex) - { - generated.Push(new ComplexNumber(complex.Real, complex.Imaginary)); - } - - public void Visit(Ast.DictNode dict) - { - IDictionary obj = new Hashtable(dict.Elements.Count); - foreach(Ast.KeyValueNode kv in dict.Elements) - { - kv.Key.Accept(this); - object key = generated.Pop(); - kv.Value.Accept(this); - object value = generated.Pop(); - obj[key] = value; - } - - if(dictToInstance==null || !obj.Contains("__class__")) - { - generated.Push(obj); - } - else - { - object result = dictToInstance(obj); - if(result==null) - generated.Push(obj); - else - generated.Push(result); - } - } - - public void Visit(Ast.ListNode list) - { - IList obj = new List(list.Elements.Count); - foreach(Ast.INode node in list.Elements) - { - node.Accept(this); - obj.Add(generated.Pop()); - } - generated.Push(obj); - } - - public void Visit(Ast.NoneNode none) - { - generated.Push(null); - } - - public void Visit(Ast.IntegerNode value) - { - generated.Push(value.Value); - } - - public void Visit(Ast.LongNode value) - { - generated.Push(value.Value); - } - - public void Visit(Ast.DoubleNode value) - { - generated.Push(value.Value); - } - - public void Visit(Ast.BooleanNode value) - { - generated.Push(value.Value); - } - - public void Visit(Ast.StringNode value) - { - generated.Push(value.Value); - } - - public void Visit(Ast.DecimalNode value) - { - generated.Push(value.Value); - } - - public void Visit(Ast.SetNode setnode) - { - HashSet obj = new HashSet(); - foreach(Ast.INode node in setnode.Elements) - { - node.Accept(this); - obj.Add(generated.Pop()); - } - generated.Push(obj); - } - - public void Visit(Ast.TupleNode tuple) - { - object[] array = new object[tuple.Elements.Count]; - int index=0; - foreach(Ast.INode node in tuple.Elements) - { - node.Accept(this); - array[index++] = generated.Pop(); - } - generated.Push(array); - } - } -} +/// +/// Serpent, a Python literal expression serializer/deserializer +/// (a.k.a. Python's ast.literal_eval in .NET) +/// +/// Copyright Irmen de Jong (irmen@razorvine.net) +/// Software license: "MIT software license". See http://opensource.org/licenses/MIT +/// + +using System; +using System.Collections.Generic; +using System.Collections; + +namespace Razorvine.Serpent +{ + /// + /// Ast nodevisitor that turns the AST into actual .NET objects (array, int, IDictionary, string, etc...) + /// + public class ObjectifyVisitor: Ast.INodeVisitor + { + private Stack generated = new Stack(); + private Func dictToInstance = null; + + /// + /// Create the visitor that converts AST in actual objects. + /// + public ObjectifyVisitor() + { + } + + /// + /// Create the visitor that converts AST in actual objects. + /// + /// functin to convert dicts to actual instances for a class, + /// instead of leaving them as dictionaries. Requires the __class__ key to be present + /// in the dict node. If it returns null, the normal processing is done. + public ObjectifyVisitor(Func dictToInstance) + { + this.dictToInstance = dictToInstance; + } + + /// + /// get the resulting object tree. + /// + public object GetObject() + { + return generated.Pop(); + } + + public void Visit(Ast.ComplexNumberNode complex) + { + generated.Push(new ComplexNumber(complex.Real, complex.Imaginary)); + } + + public void Visit(Ast.DictNode dict) + { + IDictionary obj = new Hashtable(dict.Elements.Count); + foreach(Ast.KeyValueNode kv in dict.Elements) + { + kv.Key.Accept(this); + object key = generated.Pop(); + kv.Value.Accept(this); + object value = generated.Pop(); + obj[key] = value; + } + + if(dictToInstance==null || !obj.Contains("__class__")) + { + generated.Push(obj); + } + else + { + object result = dictToInstance(obj); + if(result==null) + generated.Push(obj); + else + generated.Push(result); + } + } + + public void Visit(Ast.ListNode list) + { + IList obj = new List(list.Elements.Count); + foreach(Ast.INode node in list.Elements) + { + node.Accept(this); + obj.Add(generated.Pop()); + } + generated.Push(obj); + } + + public void Visit(Ast.NoneNode none) + { + generated.Push(null); + } + + public void Visit(Ast.IntegerNode value) + { + generated.Push(value.Value); + } + + public void Visit(Ast.LongNode value) + { + generated.Push(value.Value); + } + + public void Visit(Ast.DoubleNode value) + { + generated.Push(value.Value); + } + + public void Visit(Ast.BooleanNode value) + { + generated.Push(value.Value); + } + + public void Visit(Ast.StringNode value) + { + generated.Push(value.Value); + } + + public void Visit(Ast.DecimalNode value) + { + generated.Push(value.Value); + } + + public void Visit(Ast.SetNode setnode) + { + HashSet obj = new HashSet(); + foreach(Ast.INode node in setnode.Elements) + { + node.Accept(this); + obj.Add(generated.Pop()); + } + generated.Push(obj); + } + + public void Visit(Ast.TupleNode tuple) + { + object[] array = new object[tuple.Elements.Count]; + int index=0; + foreach(Ast.INode node in tuple.Elements) + { + node.Accept(this); + array[index++] = generated.Pop(); + } + generated.Push(array); + } + } +} diff --git a/dotnet/Serpent/ParseException.cs b/dotnet/Serpent/ParseException.cs index 8af3415..3ab1571 100644 --- a/dotnet/Serpent/ParseException.cs +++ b/dotnet/Serpent/ParseException.cs @@ -1,30 +1,30 @@ -/// -/// Serpent, a Python literal expression serializer/deserializer -/// (a.k.a. Python's ast.literal_eval in .NET) -/// -/// Copyright 2014, Irmen de Jong (irmen@razorvine.net) -/// Software license: "MIT software license". See http://opensource.org/licenses/MIT -/// - -using System; - -namespace Razorvine.Serpent -{ - /// - /// A problem occurred during parsing. - /// - public class ParseException : Exception - { - public ParseException() - { - } - - public ParseException(string message) : base(message) - { - } - - public ParseException(string message, Exception innerException) : base(message, innerException) - { - } - } +/// +/// Serpent, a Python literal expression serializer/deserializer +/// (a.k.a. Python's ast.literal_eval in .NET) +/// +/// Copyright Irmen de Jong (irmen@razorvine.net) +/// Software license: "MIT software license". See http://opensource.org/licenses/MIT +/// + +using System; + +namespace Razorvine.Serpent +{ + /// + /// A problem occurred during parsing. + /// + public class ParseException : Exception + { + public ParseException() + { + } + + public ParseException(string message) : base(message) + { + } + + public ParseException(string message, Exception innerException) : base(message, innerException) + { + } + } } \ No newline at end of file diff --git a/dotnet/Serpent/Parser.cs b/dotnet/Serpent/Parser.cs index 7e9edb1..ea1d605 100644 --- a/dotnet/Serpent/Parser.cs +++ b/dotnet/Serpent/Parser.cs @@ -2,7 +2,7 @@ /// Serpent, a Python literal expression serializer/deserializer /// (a.k.a. Python's ast.literal_eval in .NET) /// -/// Copyright 2014, Irmen de Jong (irmen@razorvine.net) +/// Copyright Irmen de Jong (irmen@razorvine.net) /// Software license: "MIT software license". See http://opensource.org/licenses/MIT /// diff --git a/dotnet/Serpent/SeekableStringReader.cs b/dotnet/Serpent/SeekableStringReader.cs index 45e6e0d..5e91128 100644 --- a/dotnet/Serpent/SeekableStringReader.cs +++ b/dotnet/Serpent/SeekableStringReader.cs @@ -2,7 +2,7 @@ /// Serpent, a Python literal expression serializer/deserializer /// (a.k.a. Python's ast.literal_eval in .NET) /// -/// Copyright 2014, Irmen de Jong (irmen@razorvine.net) +/// Copyright Irmen de Jong (irmen@razorvine.net) /// Software license: "MIT software license". See http://opensource.org/licenses/MIT /// diff --git a/dotnet/Serpent/Serializer.cs b/dotnet/Serpent/Serializer.cs index f2a7cee..c3d1bcb 100644 --- a/dotnet/Serpent/Serializer.cs +++ b/dotnet/Serpent/Serializer.cs @@ -2,7 +2,7 @@ /// Serpent, a Python literal expression serializer/deserializer /// (a.k.a. Python's ast.literal_eval in .NET) /// -/// Copyright 2014, Irmen de Jong (irmen@razorvine.net) +/// Copyright Irmen de Jong (irmen@razorvine.net) /// Software license: "MIT software license". See http://opensource.org/licenses/MIT /// diff --git a/dotnet/Serpent/Serpent.nuspec b/dotnet/Serpent/Serpent.nuspec index e23bb49..690a967 100644 --- a/dotnet/Serpent/Serpent.nuspec +++ b/dotnet/Serpent/Serpent.nuspec @@ -20,7 +20,7 @@ More info for the Python version is on Pypi: https://pypi.python.org/pypi/serpen Serpent Python literal expression serialization Custom converters now work for inheritance tree rather than just one specific class. -Added Serializer.ToBytes utility method to decode serpent base-64 encoded byte arrays. +Added Parser.ToBytes utility method to decode serpent base-64 encoded byte arrays. Copyright 2016 serialization python pyro From 2db34227c3157a44df57730de284ba3cb44ca783 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 11 Oct 2016 01:02:42 +0200 Subject: [PATCH 027/230] [maven-release-plugin] prepare release serpent-1.15 --- java/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index f101ad0..13c6c4d 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -9,7 +9,7 @@ net.razorvine serpent - 1.15-SNAPSHOT + 1.15 jar serpent @@ -67,7 +67,7 @@ It is an alternative to JSON to provide easy data integration between Java and P https://github.com/irmen/Serpent scm:git:https://github.com/irmen/Serpent.git scm:git:https://github.com/irmen/Serpent.git - HEAD + serpent-1.15 Github From 73b60e5ce453cf1f85983cf32a106303032e1eb5 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 11 Oct 2016 01:02:48 +0200 Subject: [PATCH 028/230] [maven-release-plugin] prepare for next development iteration --- java/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 13c6c4d..e68dfe3 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -9,7 +9,7 @@ net.razorvine serpent - 1.15 + 1.16-SNAPSHOT jar serpent @@ -67,7 +67,7 @@ It is an alternative to JSON to provide easy data integration between Java and P https://github.com/irmen/Serpent scm:git:https://github.com/irmen/Serpent.git scm:git:https://github.com/irmen/Serpent.git - serpent-1.15 + HEAD Github From 3f630353d739986fd222fd815b15512df836a459 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 11 Oct 2016 01:53:06 +0200 Subject: [PATCH 029/230] Parser.ToBytes fix to also recognise the actual hashtable that is produced by serpent --- dotnet/Serpent.Test/SerializeTest.cs | 11 +++++++++-- dotnet/Serpent/Parser.cs | 15 +++++++++++++++ dotnet/Serpent/Properties/AssemblyInfo.cs | 4 ++-- dotnet/Serpent/Serpent.nuspec | 2 +- 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/dotnet/Serpent.Test/SerializeTest.cs b/dotnet/Serpent.Test/SerializeTest.cs index 6ef1cba..5555a7a 100644 --- a/dotnet/Serpent.Test/SerializeTest.cs +++ b/dotnet/Serpent.Test/SerializeTest.cs @@ -319,11 +319,18 @@ public void TestBytes() string parsed = p.Parse(ser).Root.ToString(); Assert.AreEqual(39, parsed.Length); - IDictionary dict = new Dictionary { + var hashtable = new Hashtable { {"data", "YWJjZGVm"}, {"encoding", "base64"} }; - byte[] bytes2 = Parser.ToBytes(dict); + byte[] bytes2 = Parser.ToBytes(hashtable); + Assert.AreEqual(bytes, bytes2); + + var dict = new Dictionary { + {"data", "YWJjZGVm"}, + {"encoding", "base64"} + }; + bytes2 = Parser.ToBytes(dict); Assert.AreEqual(bytes, bytes2); dict["encoding"] = "base99"; diff --git a/dotnet/Serpent/Parser.cs b/dotnet/Serpent/Parser.cs index ea1d605..bae9bdc 100644 --- a/dotnet/Serpent/Parser.cs +++ b/dotnet/Serpent/Parser.cs @@ -7,6 +7,7 @@ /// using System; +using System.Collections; using System.Collections.Generic; using System.Globalization; using System.Text; @@ -610,6 +611,20 @@ double ParseDouble(string numberstr) /// If it is something else, throw an ArgumentException /// public static byte[] ToBytes(object obj) { + Hashtable hashtable = obj as Hashtable; + if(hashtable!=null) + { + string data = null; + string encoding = null; + if(hashtable.Contains("data")) data = (string)hashtable["data"]; + if(hashtable.Contains("encoding")) encoding = (string)hashtable["encoding"]; + if(data==null || "base64"!=encoding) + { + throw new ArgumentException("argument is neither bytearray nor serpent base64 encoded bytes dict"); + } + return Convert.FromBase64String(data); + } + IDictionary dict = obj as IDictionary; if(dict!=null) { diff --git a/dotnet/Serpent/Properties/AssemblyInfo.cs b/dotnet/Serpent/Properties/AssemblyInfo.cs index 66268ca..4318874 100644 --- a/dotnet/Serpent/Properties/AssemblyInfo.cs +++ b/dotnet/Serpent/Properties/AssemblyInfo.cs @@ -36,5 +36,5 @@ decoded there (using ast.literal_eval()). It can ofcourse also deserialize // // You can specify all the values or you can use the default the Revision and // Build Numbers by using the '*' as shown below: -[assembly: AssemblyVersion("1.15.0.*")] -[assembly: AssemblyFileVersion("1.15.0.0")] +[assembly: AssemblyVersion("1.15.1.*")] +[assembly: AssemblyFileVersion("1.15.1.0")] diff --git a/dotnet/Serpent/Serpent.nuspec b/dotnet/Serpent/Serpent.nuspec index 690a967..b42f635 100644 --- a/dotnet/Serpent/Serpent.nuspec +++ b/dotnet/Serpent/Serpent.nuspec @@ -2,7 +2,7 @@ Razorvine.Serpent - 1.15.0.0 + 1.15.1.0 Razorvine.Serpent Irmen de Jong Irmen de Jong From 22daf38637509f0a20242d734469cdd0e9af67d4 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 11 Oct 2016 01:55:55 +0200 Subject: [PATCH 030/230] releasenote --- dotnet/Serpent/Serpent.nuspec | 1 + 1 file changed, 1 insertion(+) diff --git a/dotnet/Serpent/Serpent.nuspec b/dotnet/Serpent/Serpent.nuspec index b42f635..f1f8035 100644 --- a/dotnet/Serpent/Serpent.nuspec +++ b/dotnet/Serpent/Serpent.nuspec @@ -21,6 +21,7 @@ More info for the Python version is on Pypi: https://pypi.python.org/pypi/serpen Custom converters now work for inheritance tree rather than just one specific class. Added Parser.ToBytes utility method to decode serpent base-64 encoded byte arrays. +(1.15.1.0: fixed Parser.ToBytes to actually work on the hashtable returned by serpent) Copyright 2016 serialization python pyro From f50023d905fce6b4b42a023711da77e9b3627117 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 11 Oct 2016 02:19:36 +0200 Subject: [PATCH 031/230] buildscripts --- dotnet/build-dotnet-microsoft.cmd | 1 + dotnet/test-dotnet.cmd | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dotnet/build-dotnet-microsoft.cmd b/dotnet/build-dotnet-microsoft.cmd index fda48e4..0b342ca 100644 --- a/dotnet/build-dotnet-microsoft.cmd +++ b/dotnet/build-dotnet-microsoft.cmd @@ -3,3 +3,4 @@ msbuild /verbosity:minimal /p:Platform="Any CPU" /p:Configuration="Release" Serp if not exist build mkdir build copy Serpent\bin\Release\*.dll build\ +copy Serpent\bin\Release\*.pdb build\ diff --git a/dotnet/test-dotnet.cmd b/dotnet/test-dotnet.cmd index 01197fa..22d4c6e 100644 --- a/dotnet/test-dotnet.cmd +++ b/dotnet/test-dotnet.cmd @@ -1,5 +1,6 @@ @echo Building new version... @call build-dotnet-microsoft.cmd @echo. -@echo Running tests... +echo "Running tests" + L:\tools\nunit2.6\nunit-console-x86 /framework:net-4.0 /nothread /noshadow .\Serpent.Test\bin\Release\Razorvine.Serpent.Test.dll From c20a775265674526c39b63fa1e6a8efba76b2930 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Thu, 13 Oct 2016 23:35:24 +0200 Subject: [PATCH 032/230] release howto --- java/mvn-release-readme.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/java/mvn-release-readme.txt b/java/mvn-release-readme.txt index 5752031..27cd217 100644 --- a/java/mvn-release-readme.txt +++ b/java/mvn-release-readme.txt @@ -6,6 +6,9 @@ $ mvn release:clean release:prepare release:perform Requires version number in the pom.xml to be "x.y-SNAPSHOT". +Finalise and publish the release via https://oss.sonatype.org/#stagingRepositories + + See also: http://java.dzone.com/articles/deploy-maven-central http://central.sonatype.org/pages/apache-maven.html#performing-a-release-deployment From a3f648ff4bc9fa497dc1cd51f12e9225dcec9463 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Fri, 21 Oct 2016 21:40:06 +0200 Subject: [PATCH 033/230] now using Azul Zulu-7 OpenJDK instead of Oracle. --- java/.classpath | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/java/.classpath b/java/.classpath index 2fa8e78..30e0063 100644 --- a/java/.classpath +++ b/java/.classpath @@ -1,23 +1,23 @@ - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + From 9b4a7925360213b625d5c0640135c0904eed62cb Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sat, 29 Oct 2016 16:26:31 +0200 Subject: [PATCH 034/230] Update LICENSE --- LICENSE | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/LICENSE b/LICENSE index 7c56fc2..abfce8a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,7 +1,6 @@ -Serpent - Python ast.literal_eval() compatible object tree serialization. -Software License, copyright, and disclaimer: +MIT License -Serpent is Copyright (c) by Irmen de Jong (irmen@razorvine.net). +Copyright (c) 2016 Irmen de Jong Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -10,18 +9,13 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - - -This is the "MIT Software License" which is OSI-certified, and GPL-compatible. -See http://www.opensource.org/licenses/mit-license.php +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From 0c150c1fb40edabed220dc00741800f61cf6f244 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sun, 30 Oct 2016 00:58:49 +0200 Subject: [PATCH 035/230] framework compatibility improvements --- dotnet/Serpent.Test/CycleTest.cs | 3 -- dotnet/Serpent.Test/Example.cs | 1 - dotnet/Serpent.Test/ParserTest.cs | 2 -- dotnet/Serpent.Test/SerializeTest.cs | 41 +++-------------------- dotnet/Serpent.Test/Serpent.Test.csproj | 9 +++-- dotnet/Serpent.Test/app.config | 2 +- dotnet/Serpent/ComplexNumber.cs | 1 - dotnet/Serpent/ObjectifyVisitor.cs | 2 +- dotnet/Serpent/Parser.cs | 6 +++- dotnet/Serpent/Properties/AssemblyInfo.cs | 8 ++--- dotnet/Serpent/Serializer.cs | 19 ++++------- dotnet/Serpent/Serpent.csproj | 16 ++++----- dotnet/Serpent/Serpent.nuspec | 6 ++-- dotnet/Serpent/Version.cs | 2 +- 14 files changed, 36 insertions(+), 82 deletions(-) diff --git a/dotnet/Serpent.Test/CycleTest.cs b/dotnet/Serpent.Test/CycleTest.cs index 5096156..ff52bd6 100644 --- a/dotnet/Serpent.Test/CycleTest.cs +++ b/dotnet/Serpent.Test/CycleTest.cs @@ -9,9 +9,6 @@ using System; using System.Collections; using System.Collections.Generic; -using System.Runtime.Serialization; -using System.Text; - using NUnit.Framework; namespace Razorvine.Serpent.Test diff --git a/dotnet/Serpent.Test/Example.cs b/dotnet/Serpent.Test/Example.cs index e70b6f1..2a11252 100644 --- a/dotnet/Serpent.Test/Example.cs +++ b/dotnet/Serpent.Test/Example.cs @@ -1,7 +1,6 @@ using System; using System.Collections; using System.Collections.Generic; -using System.Security; using System.Text; using System.Linq; using System.IO; diff --git a/dotnet/Serpent.Test/ParserTest.cs b/dotnet/Serpent.Test/ParserTest.cs index 41ef90a..a1f8895 100644 --- a/dotnet/Serpent.Test/ParserTest.cs +++ b/dotnet/Serpent.Test/ParserTest.cs @@ -11,8 +11,6 @@ using System.Collections.Generic; using System.IO; using System.Text; -using System.Xml.Serialization; - using NUnit.Framework; namespace Razorvine.Serpent.Test diff --git a/dotnet/Serpent.Test/SerializeTest.cs b/dotnet/Serpent.Test/SerializeTest.cs index 5555a7a..5e27742 100644 --- a/dotnet/Serpent.Test/SerializeTest.cs +++ b/dotnet/Serpent.Test/SerializeTest.cs @@ -7,9 +7,7 @@ /// using System; -using System.Collections; using System.Collections.Generic; -using System.Runtime.Serialization; using System.Text; using NUnit.Framework; @@ -465,10 +463,8 @@ public void TestClass() { Serializer.RegisterClass(typeof(SerializeTestClass), null); Serializer serpent = new Serializer(indent: true); - object obj = new UnserializableClass(); - Assert.Throws( ()=>serpent.Serialize(obj) ); - obj = new SerializeTestClass() { + var obj = new SerializeTestClass() { i = 99, s = "hi", x = 42 @@ -521,8 +517,6 @@ public void TestCustomClassDict() public void TestStruct() { Serializer serpent = new Serializer(indent: true); - UnserializableStruct obj; - Assert.Throws( ()=>serpent.Serialize(obj) ); var obj2 = new SerializeTestStruct() { i = 99, @@ -537,8 +531,6 @@ public void TestStruct() public void TestStruct2() { Serializer serpent = new Serializer(indent: true, namespaceInClassName: true); - UnserializableStruct obj; - Assert.Throws( ()=>serpent.Serialize(obj) ); var obj2 = new SerializeTestStruct() { i = 99, @@ -661,12 +653,7 @@ public void testAbstractBaseClassHierarchyPickler() { ConcreteSubClass c = new ConcreteSubClass(); Serializer serpent = new Serializer(); - try { - serpent.Serialize(c); - Assert.Fail("should crash"); - } catch (SerializationException x) { - Assert.IsTrue(x.Message.Contains("not serializable")); - } + serpent.Serialize(c); Serializer.RegisterClass(typeof(AbstractBaseClass), AnyClassSerializer); byte[] data = serpent.Serialize(c); @@ -679,18 +666,8 @@ public void testInterfaceHierarchyPickler() BaseClassWithInterface b = new BaseClassWithInterface(); SubClassWithInterface sub = new SubClassWithInterface(); Serializer serpent = new Serializer(); - try { - serpent.Serialize(b); - Assert.Fail("should crash"); - } catch (SerializationException x) { - Assert.IsTrue(x.Message.Contains("not serializable")); - } - try { - serpent.Serialize(sub); - Assert.Fail("should crash"); - } catch (SerializationException x) { - Assert.IsTrue(x.Message.Contains("not serializable")); - } + serpent.Serialize(b); + serpent.Serialize(sub); Serializer.RegisterClass(typeof(IBaseInterface), AnyClassSerializer); byte[] data = serpent.Serialize(b); Assert.AreEqual("{'(SUB)CLASS':'BaseClassWithInterface'}", S(strip_header(data))); @@ -699,7 +676,6 @@ public void testInterfaceHierarchyPickler() } } - [Serializable] public class SerializeTestClass { public int x; @@ -709,19 +685,10 @@ public class SerializeTestClass } - [Serializable] public struct SerializeTestStruct { public int x; public string s {get; set;} public int i {get; set;} } - - public class UnserializableClass - { - } - - public struct UnserializableStruct - { - } } diff --git a/dotnet/Serpent.Test/Serpent.Test.csproj b/dotnet/Serpent.Test/Serpent.Test.csproj index d03fa95..3fd7bc3 100644 --- a/dotnet/Serpent.Test/Serpent.Test.csproj +++ b/dotnet/Serpent.Test/Serpent.Test.csproj @@ -15,6 +15,9 @@ false 9.0.21022 2.0 + Client + v4.0 + {FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} x86 @@ -49,11 +52,11 @@ none - - ..\lib\nunit.framework.dll + + @@ -76,5 +79,5 @@ Serpent - + \ No newline at end of file diff --git a/dotnet/Serpent.Test/app.config b/dotnet/Serpent.Test/app.config index 1946b8a..0d69660 100644 --- a/dotnet/Serpent.Test/app.config +++ b/dotnet/Serpent.Test/app.config @@ -1,6 +1,6 @@  - + \ No newline at end of file diff --git a/dotnet/Serpent/ComplexNumber.cs b/dotnet/Serpent/ComplexNumber.cs index c617bf7..fc684cc 100644 --- a/dotnet/Serpent/ComplexNumber.cs +++ b/dotnet/Serpent/ComplexNumber.cs @@ -15,7 +15,6 @@ namespace Razorvine.Serpent /// /// A Complex Number class. /// -[Serializable] public class ComplexNumber { public double Real {get; private set;} diff --git a/dotnet/Serpent/ObjectifyVisitor.cs b/dotnet/Serpent/ObjectifyVisitor.cs index 5b8e456..4ba127b 100644 --- a/dotnet/Serpent/ObjectifyVisitor.cs +++ b/dotnet/Serpent/ObjectifyVisitor.cs @@ -53,7 +53,7 @@ public void Visit(Ast.ComplexNumberNode complex) public void Visit(Ast.DictNode dict) { - IDictionary obj = new Hashtable(dict.Elements.Count); + IDictionary obj = new Dictionary(dict.Elements.Count); foreach(Ast.KeyValueNode kv in dict.Elements) { kv.Key.Accept(this); diff --git a/dotnet/Serpent/Parser.cs b/dotnet/Serpent/Parser.cs index bae9bdc..0b0fbd3 100644 --- a/dotnet/Serpent/Parser.cs +++ b/dotnet/Serpent/Parser.cs @@ -7,7 +7,9 @@ /// using System; +#if !(SILVERLIGHT || WINDOWS_PHONE || PORTABLE) using System.Collections; +#endif using System.Collections.Generic; using System.Globalization; using System.Text; @@ -25,7 +27,7 @@ public class Parser /// public Ast Parse(byte[] serialized) { - return Parse(Encoding.UTF8.GetString(serialized)); + return Parse(Encoding.UTF8.GetString(serialized, 0, serialized.Length)); } /// @@ -611,6 +613,7 @@ double ParseDouble(string numberstr) /// If it is something else, throw an ArgumentException /// public static byte[] ToBytes(object obj) { +#if !(SILVERLIGHT || WINDOWS_PHONE || PORTABLE) Hashtable hashtable = obj as Hashtable; if(hashtable!=null) { @@ -624,6 +627,7 @@ public static byte[] ToBytes(object obj) { } return Convert.FromBase64String(data); } +#endif IDictionary dict = obj as IDictionary; if(dict!=null) diff --git a/dotnet/Serpent/Properties/AssemblyInfo.cs b/dotnet/Serpent/Properties/AssemblyInfo.cs index 4318874..1ffbf7e 100644 --- a/dotnet/Serpent/Properties/AssemblyInfo.cs +++ b/dotnet/Serpent/Properties/AssemblyInfo.cs @@ -26,15 +26,11 @@ decoded there (using ast.literal_eval()). It can ofcourse also deserialize [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] -// This sets the default COM visibility of types in the assembly to invisible. -// If you need to expose a type to COM, use [ComVisible(true)] on that type. -[assembly: ComVisible(false)] - // The assembly version has following format : // // Major.Minor.Build.Revision // // You can specify all the values or you can use the default the Revision and // Build Numbers by using the '*' as shown below: -[assembly: AssemblyVersion("1.15.1.*")] -[assembly: AssemblyFileVersion("1.15.1.0")] +[assembly: AssemblyVersion("1.16.0.*")] +[assembly: AssemblyFileVersion("1.16.0.0")] diff --git a/dotnet/Serpent/Serializer.cs b/dotnet/Serpent/Serializer.cs index c3d1bcb..82617fe 100644 --- a/dotnet/Serpent/Serializer.cs +++ b/dotnet/Serpent/Serializer.cs @@ -112,8 +112,8 @@ protected void Serialize(object obj, TextWriter tw, int level) else if(t.IsGenericType && t.GetGenericTypeDefinition().Equals(typeof(HashSet<>))) { IEnumerable x = (IEnumerable) obj; - ArrayList list = new ArrayList(); - foreach(var elt in x) + List list = new List(); + foreach(object elt in x) list.Add(elt); object[] setvalues = list.ToArray(); Serialize_set(setvalues, tw, level); @@ -314,7 +314,7 @@ protected void Serialize_bytes(byte[] data, TextWriter tw, int level) { // base-64 struct output string str = Convert.ToBase64String(data); - var dict = new Hashtable() { + var dict = new Dictionary() { {"data", str}, {"encoding", "base64"} }; @@ -346,7 +346,7 @@ protected void Serialize_string(string str, TextWriter tw, int level) protected void Serialize_datetime(DateTime dt, TextWriter tw, int level) { - string s = XmlConvert.ToString(dt, XmlDateTimeSerializationMode.Unspecified); + string s = dt.Millisecond == 0 ? XmlConvert.ToString(dt, "yyyy-MM-ddTHH:mm:ss") : XmlConvert.ToString(dt, "yyyy-MM-ddTHH:mm:ss.fff"); Serialize_string(s, tw, level); } @@ -379,7 +379,7 @@ protected void Serialize_exception(Exception exc, TextWriter tw, int level) className = exc.GetType().FullName; else className = exc.GetType().Name; - dict = new Hashtable() { + dict = new Dictionary { {"__class__", className}, {"__exception__", true}, {"args", new string[]{exc.Message} }, @@ -455,15 +455,8 @@ protected void Serialize_class(object obj, TextWriter tw, int level) } else { - // if it is an anonymous class type, accept it. - // any other class needs to have [Serializable] attribute bool isAnonymousClass = obj_type.Name.StartsWith("<>"); - if(!isAnonymousClass && !obj_type.IsSerializable) - { - throw new SerializationException("object of type "+obj_type.Name+" is not serializable"); - } - - dict = new Hashtable(); + dict = new Dictionary(); if(!isAnonymousClass) { // only provide the class name when it is not an anonymous class if(this.NamespaceInClassName) diff --git a/dotnet/Serpent/Serpent.csproj b/dotnet/Serpent/Serpent.csproj index 898e049..24ba936 100644 --- a/dotnet/Serpent/Serpent.csproj +++ b/dotnet/Serpent/Serpent.csproj @@ -15,6 +15,9 @@ C:\Users\Irmen\AppData\Roaming\ICSharpCode/SharpDevelop4\Settings.SourceAnalysis 10.0.0 2.0 + Client + v4.0 + {FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} x86 @@ -48,13 +51,6 @@ none - - - - - 3.5 - - @@ -70,5 +66,9 @@ - + + + + + \ No newline at end of file diff --git a/dotnet/Serpent/Serpent.nuspec b/dotnet/Serpent/Serpent.nuspec index f1f8035..d60dcc3 100644 --- a/dotnet/Serpent/Serpent.nuspec +++ b/dotnet/Serpent/Serpent.nuspec @@ -2,7 +2,7 @@ Razorvine.Serpent - 1.15.1.0 + 1.16.0.0 Razorvine.Serpent Irmen de Jong Irmen de Jong @@ -19,9 +19,7 @@ such a Python expression itself, back into the equivalent .NET datatypes. More info for the Python version is on Pypi: https://pypi.python.org/pypi/serpent Serpent Python literal expression serialization -Custom converters now work for inheritance tree rather than just one specific class. -Added Parser.ToBytes utility method to decode serpent base-64 encoded byte arrays. -(1.15.1.0: fixed Parser.ToBytes to actually work on the hashtable returned by serpent) +Framework compatibility improvements. Copyright 2016 serialization python pyro diff --git a/dotnet/Serpent/Version.cs b/dotnet/Serpent/Version.cs index 3314e82..d0e61c4 100644 --- a/dotnet/Serpent/Version.cs +++ b/dotnet/Serpent/Version.cs @@ -10,6 +10,6 @@ namespace Razorvine.Serpent { public static class LibraryVersion { - public static string Version = "1.15"; + public static string Version = "1.16"; } } From 8cef099d613c13f3f1181f401ceb9828b48dc345 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sun, 30 Oct 2016 01:30:34 +0200 Subject: [PATCH 036/230] fixed ToBytes to accept more dict types --- dotnet/Serpent.Test/SerializeTest.cs | 7 +++++++ dotnet/Serpent/Parser.cs | 15 +++++++++++++++ dotnet/Serpent/Serpent.nuspec | 1 + 3 files changed, 23 insertions(+) diff --git a/dotnet/Serpent.Test/SerializeTest.cs b/dotnet/Serpent.Test/SerializeTest.cs index 5e27742..4758499 100644 --- a/dotnet/Serpent.Test/SerializeTest.cs +++ b/dotnet/Serpent.Test/SerializeTest.cs @@ -331,6 +331,13 @@ public void TestBytes() bytes2 = Parser.ToBytes(dict); Assert.AreEqual(bytes, bytes2); + var dict2 = new Dictionary { + {"data", "YWJjZGVm"}, + {"encoding", "base64"} + }; + bytes2 = Parser.ToBytes(dict2); + Assert.AreEqual(bytes, bytes2); + dict["encoding"] = "base99"; Assert.Throws(()=>Parser.ToBytes(dict)); dict.Clear(); diff --git a/dotnet/Serpent/Parser.cs b/dotnet/Serpent/Parser.cs index 0b0fbd3..4a85379 100644 --- a/dotnet/Serpent/Parser.cs +++ b/dotnet/Serpent/Parser.cs @@ -642,6 +642,21 @@ public static byte[] ToBytes(object obj) { } return Convert.FromBase64String(data); } + IDictionary dict2 = obj as IDictionary; + if(dict2!=null) + { + object dataobj; + object encodingobj; + bool hasData = dict2.TryGetValue("data", out dataobj); + bool hasEncoding = dict2.TryGetValue("encoding", out encodingobj); + string data = (string)dataobj; + string encoding = (string)encodingobj; + if(!hasData || !hasEncoding || encoding!="base64") + { + throw new ArgumentException("argument is neither bytearray nor serpent base64 encoded bytes dict"); + } + return Convert.FromBase64String(data); + } byte[] bytearray = obj as byte[]; if(bytearray!=null) { diff --git a/dotnet/Serpent/Serpent.nuspec b/dotnet/Serpent/Serpent.nuspec index d60dcc3..0aa0684 100644 --- a/dotnet/Serpent/Serpent.nuspec +++ b/dotnet/Serpent/Serpent.nuspec @@ -20,6 +20,7 @@ More info for the Python version is on Pypi: https://pypi.python.org/pypi/serpen Serpent Python literal expression serialization Framework compatibility improvements. +ToBytes helper function can now deal with more dictionary types that are essentially the same. Copyright 2016 serialization python pyro From 3e55ce4efe1ba76ffd6a25f2094c47f32ab9f37c Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sat, 24 Dec 2016 16:59:16 +0100 Subject: [PATCH 037/230] added unit test to expose float precision loss bug on older pythons --- .travis.yml | 1 + serpent.py | 2 +- test_serpent.py | 26 ++++++++++++++++++-------- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index aead5b2..ba7ec44 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ python: - "3.3" - "3.4" - "3.5" + - "3.6" - "pypy" # Use fast travis build infrastructure explicitly diff --git a/serpent.py b/serpent.py index b1a38f6..a4617d0 100644 --- a/serpent.py +++ b/serpent.py @@ -64,7 +64,7 @@ import array import math -__version__ = "1.15" +__version__ = "1.16" __all__ = ["dump", "dumps", "load", "loads", "register_class", "unregister_class", "tobytes"] can_use_set_literals = sys.version_info >= (3, 2) # check if we can use set literals diff --git a/test_serpent.py b/test_serpent.py index 16a0b70..1b248f6 100644 --- a/test_serpent.py +++ b/test_serpent.py @@ -50,14 +50,6 @@ def test_deserialize(self): data = serpent.loads(encoded) self.assertEqual(unicodestring, data) - def test_weird_doubles(self): - values = [float('inf'), float('-inf'), float('nan')] - ser = serpent.dumps(values) - values2 = serpent.loads(ser) - self.assertEqual([float('inf'), float('-inf'), {'__class__':'float','value':'nan'}], values2) - values2 = serpent.loads(b"[1e30000,-1e30000]") - self.assertEqual([float('inf'), float('-inf')], values2) - @unittest.skipIf(sys.version_info < (3, 0), "Python 2.x ast can't parse complex") def test_weird_complex(self): c1 = complex(float('inf'), 4) @@ -550,8 +542,26 @@ def test_pickle_api(self): def test_weird_floats(self): values = [float('inf'), float('-inf'), float('nan'), complex(float('inf'), 4)] + ser = serpent.dumps(values) ser = strip_header(serpent.dumps(values)) self.assertEqual(b"[1e30000,-1e30000,{'__class__':'float','value':'nan'},(1e30000+4.0j)]", ser) + values2 = serpent.loads(ser) + self.assertEqual([float('inf'), float('-inf'), {'__class__':'float','value':'nan'}, (float('inf')+4j)], values2) + values2 = serpent.loads(b"[1e30000,-1e30000]") + self.assertEqual([float('inf'), float('-inf')], values2) + + def test_float_precision(self): + # make sure we don't lose precision when converting floats (including scientific notation) + v = serpent.loads(serpent.dumps(1.23456789)) + self.assertEqual(1.23456789, v) + v = serpent.loads(serpent.dumps(98765432123456.12345678987656)) + self.assertEqual(98765432123456.12345678987656, v) + v = serpent.loads(serpent.dumps(98765432123456.12345678987656)) + self.assertEqual(98765432123456.12345678987656, v) + v = serpent.loads(serpent.dumps(98765432123456.12345678987656e+44)) + self.assertEqual(98765432123456.12345678987656e+44, v) + v = serpent.loads(serpent.dumps((98765432123456.12345678987656e+44+665544332211.9998877665544e+33j))) + self.assertEqual((98765432123456.12345678987656e+44+665544332211.9998877665544e+33j), v) def test_tobytes(self): obj = b"test" From 2e70a18894160ef0b708476cf918624769b9d477 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sat, 24 Dec 2016 17:55:34 +0100 Subject: [PATCH 038/230] float repr instead of str to also get full precision on older pythons, fixes #19 --- serpent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serpent.py b/serpent.py index a4617d0..20cf8c8 100644 --- a/serpent.py +++ b/serpent.py @@ -331,7 +331,7 @@ def ser_builtins_float(self, float_obj, out, level): else: out.append(b"-1e30000") else: - out.append(str(float_obj).encode("ascii")) + out.append(repr(float_obj).encode("ascii")) dispatch[float] = ser_builtins_float def ser_builtins_complex(self, complex_obj, out, level): From 5d8a1d7c791c5f6f78eaee790fa329d878b65ff5 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sat, 24 Dec 2016 18:49:50 +0100 Subject: [PATCH 039/230] even better float precision test --- test_serpent.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test_serpent.py b/test_serpent.py index 1b248f6..90c17bc 100644 --- a/test_serpent.py +++ b/test_serpent.py @@ -552,10 +552,10 @@ def test_weird_floats(self): def test_float_precision(self): # make sure we don't lose precision when converting floats (including scientific notation) - v = serpent.loads(serpent.dumps(1.23456789)) - self.assertEqual(1.23456789, v) - v = serpent.loads(serpent.dumps(98765432123456.12345678987656)) - self.assertEqual(98765432123456.12345678987656, v) + v = serpent.loads(serpent.dumps(1.2345678987654321)) + self.assertEqual(1.2345678987654321, v) + v = serpent.loads(serpent.dumps(5555.12345678987656)) + self.assertEqual(5555.12345678987656, v) v = serpent.loads(serpent.dumps(98765432123456.12345678987656)) self.assertEqual(98765432123456.12345678987656, v) v = serpent.loads(serpent.dumps(98765432123456.12345678987656e+44)) From 1c50869a08c272e7ee454c979ad939c39ef2dc4d Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sat, 24 Dec 2016 18:50:17 +0100 Subject: [PATCH 040/230] enum type unittest added --- test_serpent.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test_serpent.py b/test_serpent.py index 90c17bc..c36d07b 100644 --- a/test_serpent.py +++ b/test_serpent.py @@ -563,6 +563,17 @@ def test_float_precision(self): v = serpent.loads(serpent.dumps((98765432123456.12345678987656e+44+665544332211.9998877665544e+33j))) self.assertEqual((98765432123456.12345678987656e+44+665544332211.9998877665544e+33j), v) + @unittest.skipIf(sys.version_info < (3, 4), "needs python 3.4 to test enum type") + def test_enums(self): + import enum + class Animal(enum.Enum): + BEE = 1 + CAT = 2 + DOG = 3 + print(Animal.CAT) + v = serpent.loads(serpent.dumps(Animal.CAT)) + self.assertEqual("Animal.CAT", v) + def test_tobytes(self): obj = b"test" self.assertIs(obj, serpent.tobytes(obj)) From 5c3cd3fd1ed74b00ce697bd7bbba3a160d95e2e6 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sat, 24 Dec 2016 18:52:29 +0100 Subject: [PATCH 041/230] java and .net tests added that expose problems with floating point precision --- dotnet/Serpent.Test/ParserTest.cs | 1533 ++++++++------- dotnet/Serpent.Test/SerializeTest.cs | 1402 ++++++------- .../net/razorvine/serpent/LibraryVersion.java | 24 +- .../net/razorvine/serpent/package-info.java | 20 +- .../razorvine/serpent/test/ParserTest.java | 1742 +++++++++-------- 5 files changed, 2408 insertions(+), 2313 deletions(-) diff --git a/dotnet/Serpent.Test/ParserTest.cs b/dotnet/Serpent.Test/ParserTest.cs index a1f8895..020fefc 100644 --- a/dotnet/Serpent.Test/ParserTest.cs +++ b/dotnet/Serpent.Test/ParserTest.cs @@ -1,744 +1,791 @@ -/// -/// Serpent, a Python literal expression serializer/deserializer -/// (a.k.a. Python's ast.literal_eval in .NET) -/// -/// Copyright Irmen de Jong (irmen@razorvine.net) -/// Software license: "MIT software license". See http://opensource.org/licenses/MIT -/// - -using System; -using System.Collections; -using System.Collections.Generic; -using System.IO; -using System.Text; -using NUnit.Framework; - -namespace Razorvine.Serpent.Test -{ - [TestFixture] - public class ParserTest - { - [Test] - public void TestBasic() - { - Parser p = new Parser(); - Assert.IsNull(p.Parse((string)null).Root); - Assert.IsNull(p.Parse("").Root); - Assert.IsNotNull(p.Parse("# comment\n42\n").Root); - } - - [Test] - public void TestComments() - { - Parser p = new Parser(); - - Ast ast = p.Parse("[ 1, 2 ]"); // no header whatsoever - var visitor = new ObjectifyVisitor(); - ast.Accept(visitor); - Object obj = visitor.GetObject(); - Assert.AreEqual(new int[] {1,2}, obj); - - ast = p.Parse(@"# serpent utf-8 python2.7 -[ 1, 2, - # some comments here - 3, 4] # more here -# and here. -"); - visitor = new ObjectifyVisitor(); - ast.Accept(visitor); - obj = visitor.GetObject(); - Assert.AreEqual(new int[] {1,2,3,4}, obj); - } - - [Test] - public void TestPrimitives() - { - Parser p = new Parser(); - Assert.AreEqual(new Ast.IntegerNode(42), p.Parse("42").Root); - Assert.AreEqual(new Ast.IntegerNode(-42), p.Parse("-42").Root); - Assert.AreEqual(new Ast.DoubleNode(42.331), p.Parse("42.331").Root); - Assert.AreEqual(new Ast.DoubleNode(-42.331), p.Parse("-42.331").Root); - Assert.AreEqual(new Ast.DoubleNode(-1.2e19), p.Parse("-1.2e+19").Root); - Assert.AreEqual(new Ast.DoubleNode(0.0004), p.Parse("4e-4").Root); - Assert.AreEqual(new Ast.DoubleNode(40000), p.Parse("4e4").Root); - Assert.AreEqual(new Ast.BooleanNode(true), p.Parse("True").Root); - Assert.AreEqual(new Ast.BooleanNode(false), p.Parse("False").Root); - Assert.AreEqual(Ast.NoneNode.Instance, p.Parse("None").Root); - - // long ints - Assert.AreEqual(new Ast.DecimalNode(123456789123456789123456789M), p.Parse("123456789123456789123456789").Root); - Assert.AreNotEqual(new Ast.LongNode(52), p.Parse("52").Root); - Assert.AreEqual(new Ast.LongNode(123456789123456789L), p.Parse("123456789123456789").Root); - Assert.Throws(()=>p.Parse("123456789123456789123456789123456789")); // overflow - } - - [Test] - public void TestWeirdFloats() - { - Parser p = new Parser(); - var tuple = (Ast.TupleNode) p.Parse("(1e30000,-1e30000,(1e30000+3.4j),{'__class__':'float','value':'nan'})").Root; - Assert.AreEqual(4, tuple.Elements.Count); - var d = (Ast.DoubleNode) tuple.Elements[0]; - Assert.IsTrue(double.IsPositiveInfinity(d.Value)); - d = (Ast.DoubleNode) tuple.Elements[1]; - Assert.IsTrue(double.IsNegativeInfinity(d.Value)); - var c = (Ast.ComplexNumberNode) tuple.Elements[2]; - Assert.IsTrue(double.IsInfinity(c.Real)); - Assert.AreEqual(3.4, c.Imaginary, 0); - d = (Ast.DoubleNode) tuple.Elements[3]; - Assert.IsTrue(Double.IsNaN(d.Value)); - } - - [Test] - public void TestEquality() - { - Ast.INode n1, n2; - - n1 = new Ast.IntegerNode(42); - n2 = new Ast.IntegerNode(42); - Assert.AreEqual(n1, n2); - n2 = new Ast.IntegerNode(43); - Assert.AreNotEqual(n1, n2); - - n1 = new Ast.StringNode("foo"); - n2 = new Ast.StringNode("foo"); - Assert.AreEqual(n1, n2); - n2 = new Ast.StringNode("bar"); - Assert.AreNotEqual(n1, n2); - - n1 = new Ast.ComplexNumberNode() { - Real=1.1, - Imaginary=2.2 - }; - n2 = new Ast.ComplexNumberNode() { - Real=1.1, - Imaginary=2.2 - }; - Assert.AreEqual(n1, n2); - n2 = new Ast.ComplexNumberNode() { - Real=1.1, - Imaginary=3.3 - }; - Assert.AreNotEqual(n1, n2); - - n1=new Ast.KeyValueNode() { - Key=new Ast.IntegerNode(42), - Value=new Ast.IntegerNode(42) - }; - n2=new Ast.KeyValueNode() { - Key=new Ast.IntegerNode(42), - Value=new Ast.IntegerNode(42) - }; - Assert.AreEqual(n1, n2); - n1=new Ast.KeyValueNode() { - Key=new Ast.IntegerNode(43), - Value=new Ast.IntegerNode(43) - }; - Assert.AreNotEqual(n1,n2); - - n1=Ast.NoneNode.Instance; - n2=Ast.NoneNode.Instance; - Assert.AreEqual(n1, n2); - n2=new Ast.IntegerNode(42); - Assert.AreNotEqual(n1, n2); - - n1=new Ast.DictNode() { - Elements=new List() { - new Ast.KeyValueNode() { - Key=new Ast.IntegerNode(42), - Value=new Ast.IntegerNode(42) - } - } - }; - n2=new Ast.DictNode() { - Elements=new List() { - new Ast.KeyValueNode() { - Key=new Ast.IntegerNode(42), - Value=new Ast.IntegerNode(42) - } - } - }; - Assert.AreEqual(n1, n2); - n2=new Ast.DictNode() { - Elements=new List() { - new Ast.KeyValueNode() { - Key=new Ast.IntegerNode(42), - Value=new Ast.IntegerNode(43) - } - } - }; - Assert.AreNotEqual(n1, n2); - - n1=new Ast.ListNode() { - Elements=new List() { - new Ast.IntegerNode(42) - } - }; - n2=new Ast.ListNode() { - Elements=new List() { - new Ast.IntegerNode(42) - } - }; - Assert.AreEqual(n1,n2); - n2=new Ast.ListNode() { - Elements=new List() { - new Ast.IntegerNode(43) - } - }; - Assert.AreNotEqual(n1,n2); - - n1=new Ast.SetNode() { - Elements=new List() { - new Ast.IntegerNode(42) - } - }; - n2=new Ast.SetNode() { - Elements=new List() { - new Ast.IntegerNode(42) - } - }; - Assert.AreEqual(n1,n2); - n2=new Ast.SetNode() { - Elements=new List() { - new Ast.IntegerNode(43) - } - }; - Assert.AreNotEqual(n1,n2); - - n1=new Ast.TupleNode() { - Elements=new List() { - new Ast.IntegerNode(42) - } - }; - n2=new Ast.TupleNode() { - Elements=new List() { - new Ast.IntegerNode(42) - } - }; - Assert.AreEqual(n1,n2); - n2=new Ast.TupleNode() { - Elements=new List() { - new Ast.IntegerNode(43) - } - }; - Assert.AreNotEqual(n1,n2); - - } - - [Test] - public void TestDictEquality() - { - Ast.DictNode dict1 = new Ast.DictNode(); - Ast.KeyValueNode kv=new Ast.KeyValueNode() { - Key=new Ast.StringNode("key1"), - Value=new Ast.IntegerNode(42) - }; - dict1.Elements.Add(kv); - kv=new Ast.KeyValueNode() { - Key=new Ast.StringNode("key2"), - Value=new Ast.IntegerNode(43) - }; - dict1.Elements.Add(kv); - kv=new Ast.KeyValueNode() { - Key=new Ast.StringNode("key3"), - Value=new Ast.IntegerNode(44) - }; - dict1.Elements.Add(kv); - - Ast.DictNode dict2 = new Ast.DictNode(); - kv=new Ast.KeyValueNode(){ - Key=new Ast.StringNode("key2"), - Value=new Ast.IntegerNode(43) - }; - dict2.Elements.Add(kv); - kv=new Ast.KeyValueNode() { - Key=new Ast.StringNode("key3"), - Value=new Ast.IntegerNode(44) - }; - dict2.Elements.Add(kv); - kv=new Ast.KeyValueNode() { - Key=new Ast.StringNode("key1"), - Value=new Ast.IntegerNode(42) - }; - dict2.Elements.Add(kv); - - Assert.AreEqual(dict1, dict2); - kv=new Ast.KeyValueNode() { - Key=new Ast.StringNode("key4"), - Value=new Ast.IntegerNode(45) - }; - dict2.Elements.Add(kv); - Assert.AreNotEqual(dict1, dict2); - } - - [Test] - public void TestSetEquality() - { - Ast.SetNode set1 = new Ast.SetNode(); - set1.Elements.Add(new Ast.IntegerNode(1)); - set1.Elements.Add(new Ast.IntegerNode(2)); - set1.Elements.Add(new Ast.IntegerNode(3)); - - Ast.SetNode set2 = new Ast.SetNode(); - set2.Elements.Add(new Ast.IntegerNode(2)); - set2.Elements.Add(new Ast.IntegerNode(3)); - set2.Elements.Add(new Ast.IntegerNode(1)); - - Assert.AreEqual(set1, set2); - set2.Elements.Add(new Ast.IntegerNode(0)); - Assert.AreNotEqual(set1, set2); - } - - [Test] - public void TestPrintSingle() - { - Parser p = new Parser(); - - // primitives - Assert.AreEqual("42", p.Parse("42").Root.ToString()); - Assert.AreEqual("-42.331", p.Parse("-42.331").Root.ToString()); - Assert.AreEqual("-42.0", p.Parse("-42.0").Root.ToString()); - Assert.AreEqual("-2E+20", p.Parse("-2E20").Root.ToString()); - Assert.AreEqual("2.0", p.Parse("2.0").Root.ToString()); - Assert.AreEqual("1.2E+19", p.Parse("1.2e19").Root.ToString()); - Assert.AreEqual("True", p.Parse("True").Root.ToString()); - Assert.AreEqual("'hello'", p.Parse("'hello'").Root.ToString()); - Assert.AreEqual("'\\n'", p.Parse("'\n'").Root.ToString()); - Assert.AreEqual("'\\''", p.Parse("'\\''").Root.ToString()); - Assert.AreEqual("'\"'", p.Parse("'\\\"'").Root.ToString()); - Assert.AreEqual("'\"'", p.Parse("'\"'").Root.ToString()); - Assert.AreEqual("'\\\\'", p.Parse("'\\\\'").Root.ToString()); - Assert.AreEqual("None", p.Parse("None").Root.ToString()); - string ustr = "'\u20ac\u2603'"; - Assert.AreEqual(ustr, p.Parse(ustr).Root.ToString()); - - // complex - Assert.AreEqual("(0+2j)", p.Parse("2j").Root.ToString()); - Assert.AreEqual("(-1.1-2.2j)", p.Parse("(-1.1-2.2j)").Root.ToString()); - Assert.AreEqual("(1.1+2.2j)", p.Parse("(1.1+2.2j)").Root.ToString()); - - // long int - Assert.AreEqual("123456789123456789123456789", p.Parse("123456789123456789123456789").Root.ToString()); - } - - [Test] - public void TestPrintSeq() - { - Parser p=new Parser(); - - //tuple - Assert.AreEqual("()", p.Parse("()").Root.ToString()); - Assert.AreEqual("(42,)", p.Parse("(42,)").Root.ToString()); - Assert.AreEqual("(42,43)", p.Parse("(42,43)").Root.ToString()); - - // list - Assert.AreEqual("[]", p.Parse("[]").Root.ToString()); - Assert.AreEqual("[42]", p.Parse("[42]").Root.ToString()); - Assert.AreEqual("[42,43]", p.Parse("[42,43]").Root.ToString()); - - // set - Assert.AreEqual("{42}", p.Parse("{42}").Root.ToString()); - Assert.AreEqual("{42,43}", p.Parse("{42,43,43,43}").Root.ToString()); - - // dict - Assert.AreEqual("{}", p.Parse("{}").Root.ToString()); - Assert.AreEqual("{'a':42}", p.Parse("{'a': 42}").Root.ToString()); - Assert.AreEqual("{'a':42,'b':43}", p.Parse("{'a': 42, 'b': 43}").Root.ToString()); - Assert.AreEqual("{'a':42,'b':45}", p.Parse("{'a': 42, 'b': 43, 'b': 44, 'b': 45}").Root.ToString()); - } - - [Test] - public void TestInvalidPrimitives() - { - Parser p = new Parser(); - Assert.Throws(()=>p.Parse("1+2")); - Assert.Throws(()=>p.Parse("1-2")); - Assert.Throws(()=>p.Parse("1.1+2.2")); - Assert.Throws(()=>p.Parse("1.1-2.2")); - Assert.Throws(()=>p.Parse("True+2")); - Assert.Throws(()=>p.Parse("False-2")); - Assert.Throws(()=>p.Parse("3j+2")); - Assert.Throws(()=>p.Parse("3j-2")); - Assert.Throws(()=>p.Parse("None+2")); - Assert.Throws(()=>p.Parse("None-2")); - } - - [Test] - public void TestComplex() - { - Parser p = new Parser(); - var cplx = new Ast.ComplexNumberNode() { - Real = 4.2, - Imaginary = 3.2 - }; - var cplx2 = new Ast.ComplexNumberNode() { - Real = 4.2, - Imaginary = 99 - }; - Assert.AreNotEqual(cplx, cplx2); - cplx2.Imaginary = 3.2; - Assert.AreEqual(cplx, cplx2); - - Assert.AreEqual(cplx, p.Parse("(4.2+3.2j)").Root); - cplx.Real = 0; - Assert.AreEqual(cplx, p.Parse("(0+3.2j)").Root); - Assert.AreEqual(cplx, p.Parse("3.2j").Root); - Assert.AreEqual(cplx, p.Parse("+3.2j").Root); - cplx.Imaginary = -3.2; - Assert.AreEqual(cplx, p.Parse("-3.2j").Root); - cplx.Real = -9.9; - Assert.AreEqual(cplx, p.Parse("(-9.9-3.2j)").Root); - - cplx.Real = 2; - cplx.Imaginary = 3; - Assert.AreEqual(cplx, p.Parse("(2+3j)").Root); - cplx.Imaginary = -3; - Assert.AreEqual(cplx, p.Parse("(2-3j)").Root); - cplx.Real = 0; - Assert.AreEqual(cplx, p.Parse("-3j").Root); - } - - [Test] - public void TestPrimitivesStuffAtEnd() - { - Parser p = new Parser(); - Assert.AreEqual(new Ast.IntegerNode(42), p.ParseSingle(new SeekableStringReader("42@"))); - Assert.AreEqual(new Ast.DoubleNode(42.331), p.ParseSingle(new SeekableStringReader("42.331@"))); - Assert.AreEqual(new Ast.BooleanNode(true), p.ParseSingle(new SeekableStringReader("True@"))); - Assert.AreEqual(Ast.NoneNode.Instance, p.ParseSingle(new SeekableStringReader("None@"))); - var cplx = new Ast.ComplexNumberNode() { - Real = 4, - Imaginary = 3 - }; - Assert.AreEqual(cplx, p.ParseSingle(new SeekableStringReader("(4+3j)@"))); - cplx.Real=0; - Assert.AreEqual(cplx, p.ParseSingle(new SeekableStringReader("3j@"))); - } - - [Test] - public void TestStrings() - { - Parser p = new Parser(); - Assert.AreEqual(new Ast.StringNode("hello"), p.Parse("'hello'").Root); - Assert.AreEqual(new Ast.StringNode("hello"), p.Parse("\"hello\"").Root); - Assert.AreEqual(new Ast.StringNode("\\"), p.Parse("'\\\\'").Root); - Assert.AreEqual(new Ast.StringNode("\\"), p.Parse("\"\\\\\"").Root); - Assert.AreEqual(new Ast.StringNode("'"), p.Parse("\"'\"").Root); - Assert.AreEqual(new Ast.StringNode("\""), p.Parse("'\"'").Root); - Assert.AreEqual(new Ast.StringNode("tab\tnewline\n."), p.Parse("'tab\\tnewline\\n.'").Root); - } - - [Test] - public void TestUnicode() - { - Parser p = new Parser(); - string str = "'\u20ac\u2603'"; - Assert.AreEqual(0x20ac, str[1]); - Assert.AreEqual(0x2603, str[2]); - byte[] bytes = Encoding.UTF8.GetBytes(str); - - string value = "\u20ac\u2603"; - Assert.AreEqual(new Ast.StringNode(value), p.Parse(str).Root); - Assert.AreEqual(new Ast.StringNode(value), p.Parse(bytes).Root); - } - - [Test] - public void TestWhitespace() - { - Parser p = new Parser(); - Assert.AreEqual(new Ast.IntegerNode(42), p.Parse(" 42 ").Root); - Assert.AreEqual(new Ast.IntegerNode(42), p.Parse(" 42 ").Root); - Assert.AreEqual(new Ast.IntegerNode(42), p.Parse("\t42\r\n").Root); - Assert.AreEqual(new Ast.IntegerNode(42), p.Parse(" \t 42 \r \n ").Root); - Assert.AreEqual(new Ast.StringNode(" string value "), p.Parse(" ' string value ' ").Root); - Assert.Throws(()=>p.Parse(" ( 42 , ( 'x', 'y' ) ")); // missing tuple close ) - Ast ast = p.Parse(" ( 42 , ( 'x', 'y' ) ) "); - Ast.TupleNode tuple = (Ast.TupleNode) ast.Root; - Assert.AreEqual(new Ast.IntegerNode(42), tuple.Elements[0]); - tuple = (Ast.TupleNode) tuple.Elements[1]; - Assert.AreEqual(new Ast.StringNode("x"), tuple.Elements[0]); - Assert.AreEqual(new Ast.StringNode("y"), tuple.Elements[1]); - - p.Parse(" ( 52 , ) "); - p.Parse(" [ 52 ] "); - p.Parse(" { 'a' : 42 } "); - p.Parse(" { 52 } "); - } - - [Test] - public void TestTuple() - { - Parser p = new Parser(); - Ast.TupleNode tuple = new Ast.TupleNode(); - Ast.TupleNode tuple2 = new Ast.TupleNode(); - Assert.AreEqual(tuple, tuple2); - - tuple.Elements.Add(new Ast.IntegerNode(42)); - tuple2.Elements.Add(new Ast.IntegerNode(99)); - Assert.AreNotEqual(tuple, tuple2); - tuple2.Elements.Clear(); - tuple2.Elements.Add(new Ast.IntegerNode(42)); - Assert.AreEqual(tuple, tuple2); - tuple2.Elements.Add(new Ast.IntegerNode(43)); - tuple2.Elements.Add(new Ast.IntegerNode(44)); - Assert.AreNotEqual(tuple, tuple2); - - Assert.AreEqual(new Ast.TupleNode(), p.Parse("()").Root); - Assert.AreEqual(tuple, p.Parse("(42,)").Root); - Assert.AreEqual(tuple2, p.Parse("( 42,43, 44 )").Root); - - Assert.Throws(()=>p.Parse("(42,43]")); - Assert.Throws(()=>p.Parse("()@")); - Assert.Throws(()=>p.Parse("(42,43)@")); - } - - [Test] - public void TestList() - { - Parser p = new Parser(); - Ast.ListNode list = new Ast.ListNode(); - Ast.ListNode list2 = new Ast.ListNode(); - Assert.AreEqual(list, list2); - - list.Elements.Add(new Ast.IntegerNode(42)); - list2.Elements.Add(new Ast.IntegerNode(99)); - Assert.AreNotEqual(list, list2); - list2.Elements.Clear(); - list2.Elements.Add(new Ast.IntegerNode(42)); - Assert.AreEqual(list, list2); - list2.Elements.Add(new Ast.IntegerNode(43)); - list2.Elements.Add(new Ast.IntegerNode(44)); - Assert.AreNotEqual(list, list2); - - Assert.AreEqual(new Ast.ListNode(), p.Parse("[]").Root); - Assert.AreEqual(list, p.Parse("[42]").Root); - Assert.AreEqual(list2, p.Parse("[ 42,43, 44 ]").Root); - - Assert.Throws(()=>p.Parse("[42,43}")); - Assert.Throws(()=>p.Parse("[]@")); - Assert.Throws(()=>p.Parse("[42,43]@")); - } - - [Test] - public void TestSet() - { - Parser p = new Parser(); - Ast.SetNode set1 = new Ast.SetNode(); - Ast.SetNode set2 = new Ast.SetNode(); - Assert.AreEqual(set1, set2); - - set1.Elements.Add(new Ast.IntegerNode(42)); - set2.Elements.Add(new Ast.IntegerNode(99)); - Assert.AreNotEqual(set1, set2); - set2.Elements.Clear(); - set2.Elements.Add(new Ast.IntegerNode(42)); - Assert.AreEqual(set1, set2); - - set2.Elements.Add(new Ast.IntegerNode(43)); - set2.Elements.Add(new Ast.IntegerNode(44)); - Assert.AreNotEqual(set1, set2); - - Assert.AreEqual(set1, p.Parse("{42}").Root); - Assert.AreEqual(set2, p.Parse("{ 42,43, 44 }").Root); - - Assert.Throws(()=>p.Parse("{42,43]")); - Assert.Throws(()=>p.Parse("{42,43}@")); - - set1 = p.Parse("{'first','second','third','fourth','fifth','second', 'first', 'third', 'third' }").Root as Ast.SetNode; - Assert.AreEqual("'first'", set1.Elements[0].ToString()); - Assert.AreEqual("'second'", set1.Elements[1].ToString()); - Assert.AreEqual("'third'", set1.Elements[2].ToString()); - Assert.AreEqual("'fourth'", set1.Elements[3].ToString()); - Assert.AreEqual("'fifth'", set1.Elements[4].ToString()); - Assert.AreEqual(5, set1.Elements.Count); - } - - [Test] - public void TestDict() - { - Parser p = new Parser(); - Ast.DictNode dict1 = new Ast.DictNode(); - Ast.DictNode dict2 = new Ast.DictNode(); - Assert.AreEqual(dict1, dict2); - - Ast.KeyValueNode kv1 = new Ast.KeyValueNode { Key=new Ast.StringNode("key"), Value=new Ast.IntegerNode(42) }; - Ast.KeyValueNode kv2 = new Ast.KeyValueNode { Key=new Ast.StringNode("key"), Value=new Ast.IntegerNode(99) }; - Assert.AreNotEqual(kv1, kv2); - kv2.Value = new Ast.IntegerNode(42); - Assert.AreEqual(kv1, kv2); - - dict1.Elements.Add(new Ast.KeyValueNode { Key=new Ast.StringNode("key1"), Value=new Ast.IntegerNode(42) }); - dict2.Elements.Add(new Ast.KeyValueNode { Key=new Ast.StringNode("key1"), Value=new Ast.IntegerNode(99) }); - Assert.AreNotEqual(dict1, dict2); - dict2.Elements.Clear(); - dict2.Elements.Add(new Ast.KeyValueNode { Key=new Ast.StringNode("key1"), Value=new Ast.IntegerNode(42) }); - Assert.AreEqual(dict1, dict2); - - dict2.Elements.Add(new Ast.KeyValueNode { Key=new Ast.StringNode("key2"), Value=new Ast.IntegerNode(43) }); - dict2.Elements.Add(new Ast.KeyValueNode { Key=new Ast.StringNode("key3"), Value=new Ast.IntegerNode(44) }); - Assert.AreNotEqual(dict1, dict2); - - Assert.AreEqual(new Ast.DictNode(), p.Parse("{}").Root); - Assert.AreEqual(dict1, p.Parse("{'key1': 42}").Root); - Assert.AreEqual(dict2, p.Parse("{'key1': 42, 'key2': 43, 'key3':44}").Root); - - Assert.Throws(()=>p.Parse("{'key': 42]")); - Assert.Throws(()=>p.Parse("{}@")); - Assert.Throws(()=>p.Parse("{'key': 42}@")); - - dict1 = p.Parse("{'a': 1, 'b': 2, 'c': 3, 'c': 4, 'c': 5, 'c': 6}").Root as Ast.DictNode; - Assert.AreEqual("'a':1", dict1.Elements[0].ToString()); - Assert.AreEqual("'b':2", dict1.Elements[1].ToString()); - Assert.AreEqual("'c':6", dict1.Elements[2].ToString()); - Assert.AreEqual(3, dict1.Elements.Count); - } - - [Test] - public void TestFile() - { - Parser p = new Parser(); - byte[] ser=File.ReadAllBytes("testserpent.utf8.bin"); - Ast ast = p.Parse(ser); - - string expr = ast.ToString(); - Ast ast2 = p.Parse(expr); - string expr2 = ast2.ToString(); - Assert.AreEqual(expr, expr2); - - StringBuilder sb= new StringBuilder(); - Walk(ast.Root, sb); - string walk1 = sb.ToString(); - sb= new StringBuilder(); - Walk(ast2.Root, sb); - string walk2 = sb.ToString(); - Assert.AreEqual(walk1, walk2); - - // @TODO Assert.AreEqual(ast.Root, ast2.Root); - ast = p.Parse(expr2); - // @TODO Assert.AreEqual(ast.Root, ast2.Root); - } - - [Test] - [Ignore("can't yet get the ast to compare equal on mono")] - public void TestAstEquals() - { - Parser p = new Parser (); - byte[] ser = File.ReadAllBytes ("testserpent.utf8.bin"); - Ast ast = p.Parse(ser); - Ast ast2 = p.Parse(ser); - Assert.AreEqual(ast.Root, ast2.Root); - } - - public void Walk(Ast.INode node, StringBuilder sb) - { - if(node is Ast.SequenceNode) - { - sb.AppendLine(string.Format("{0} (seq)", node.GetType())); - Ast.SequenceNode seq = (Ast.SequenceNode)node; - foreach(Ast.INode child in seq.Elements) { - Walk(child, sb); - } - } - else - sb.AppendLine(string.Format("{0} = {1}", node.GetType(), node.ToString())); - } - - [Test] - public void TestTrailingCommas() - { - Parser p = new Parser(); - Ast.INode result; - result = p.Parse("[1,2,3, ]").Root; - result = p.Parse("[1,2,3 , ]").Root; - result = p.Parse("[1,2,3,]").Root; - Assert.AreEqual("[1,2,3]", result.ToString()); - result = p.Parse("(1,2,3, )").Root; - result = p.Parse("(1,2,3 , )").Root; - result = p.Parse("(1,2,3,)").Root; - Assert.AreEqual("(1,2,3)", result.ToString()); - - // for dict and set the asserts are a bit more complex - // we cannot simply convert to string because the order of elts is undefined. - - result = p.Parse("{'a':1, 'b':2, 'c':3, }").Root; - result = p.Parse("{'a':1, 'b':2, 'c':3 , }").Root; - result = p.Parse("{'a':1, 'b':2, 'c':3,}").Root; - Ast.DictNode dict = (Ast.DictNode) result; - var items = dict.ElementsAsSet(); - Assert.IsTrue(items.Contains(new Ast.KeyValueNode(new Ast.StringNode("a"), new Ast.IntegerNode(1)))); - Assert.IsTrue(items.Contains(new Ast.KeyValueNode(new Ast.StringNode("b"), new Ast.IntegerNode(2)))); - Assert.IsTrue(items.Contains(new Ast.KeyValueNode(new Ast.StringNode("c"), new Ast.IntegerNode(3)))); - result = p.Parse("{1,2,3, }").Root; - result = p.Parse("{1,2,3 , }").Root; - result = p.Parse("{1,2,3,}").Root; - Ast.SetNode set = (Ast.SetNode) result; - items = set.ElementsAsSet(); - Assert.IsTrue(items.Contains(new Ast.IntegerNode(1))); - Assert.IsTrue(items.Contains(new Ast.IntegerNode(2))); - Assert.IsTrue(items.Contains(new Ast.IntegerNode(3))); - Assert.IsFalse(items.Contains(new Ast.IntegerNode(4))); - } - } - - [TestFixture] - public class VisitorTest - { - [Test] - public void TestObjectify() - { - Parser p = new Parser(); - byte[] ser=File.ReadAllBytes("testserpent.utf8.bin"); - Ast ast = p.Parse(ser); - var visitor = new ObjectifyVisitor(); - ast.Accept(visitor); - object thing = visitor.GetObject(); - - IDictionary dict = (IDictionary) thing; - Assert.AreEqual(11, dict.Count); - IList list = dict["numbers"] as IList; - Assert.AreEqual(4, list.Count); - Assert.AreEqual(999.1234, list[1]); - Assert.AreEqual(new ComplexNumber(-3, 8), list[3]); - string euro = dict["unicode"] as string; - Assert.AreEqual("\u20ac", euro); - IDictionary exc = (IDictionary)dict["exc"]; - object[] args = (object[]) exc["args"]; - Assert.AreEqual("fault", args[0]); - Assert.AreEqual("ZeroDivisionError", exc["__class__"]); - } - - object ZerodivisionFromDict(IDictionary dict) - { - string classname = (string)dict["__class__"]; - if(classname=="ZeroDivisionError") - { - object[] args = (object[]) dict["args"]; - return new DivideByZeroException((string)args[0]); - } - return null; - } - - [Test] - public void TestObjectifyDictToClass() - { - Parser p = new Parser(); - byte[] ser=File.ReadAllBytes("testserpent.utf8.bin"); - Ast ast = p.Parse(ser); - - var visitor = new ObjectifyVisitor(ZerodivisionFromDict); - ast.Accept(visitor); - object thing = visitor.GetObject(); - - IDictionary dict = (IDictionary) thing; - Assert.AreEqual(11, dict.Count); - DivideByZeroException ex = (DivideByZeroException) dict["exc"]; - Assert.AreEqual("fault", ex.Message); - - thing = ast.GetData(ZerodivisionFromDict); - dict = (IDictionary) thing; - Assert.AreEqual(11, dict.Count); - ex = (DivideByZeroException) dict["exc"]; - Assert.AreEqual("fault", ex.Message); - } - } +/// +/// Serpent, a Python literal expression serializer/deserializer +/// (a.k.a. Python's ast.literal_eval in .NET) +/// +/// Copyright Irmen de Jong (irmen@razorvine.net) +/// Software license: "MIT software license". See http://opensource.org/licenses/MIT +/// + +using System; +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Text; +using NUnit.Framework; + +namespace Razorvine.Serpent.Test +{ + [TestFixture] + public class ParserTest + { + [Test] + public void TestBasic() + { + Parser p = new Parser(); + Assert.IsNull(p.Parse((string)null).Root); + Assert.IsNull(p.Parse("").Root); + Assert.IsNotNull(p.Parse("# comment\n42\n").Root); + } + + [Test] + public void TestComments() + { + Parser p = new Parser(); + + Ast ast = p.Parse("[ 1, 2 ]"); // no header whatsoever + var visitor = new ObjectifyVisitor(); + ast.Accept(visitor); + Object obj = visitor.GetObject(); + Assert.AreEqual(new int[] {1,2}, obj); + + ast = p.Parse(@"# serpent utf-8 python2.7 +[ 1, 2, + # some comments here + 3, 4] # more here +# and here. +"); + visitor = new ObjectifyVisitor(); + ast.Accept(visitor); + obj = visitor.GetObject(); + Assert.AreEqual(new int[] {1,2,3,4}, obj); + } + + [Test] + public void TestPrimitives() + { + Parser p = new Parser(); + Assert.AreEqual(new Ast.IntegerNode(42), p.Parse("42").Root); + Assert.AreEqual(new Ast.IntegerNode(-42), p.Parse("-42").Root); + Assert.AreEqual(new Ast.DoubleNode(42.331), p.Parse("42.331").Root); + Assert.AreEqual(new Ast.DoubleNode(-42.331), p.Parse("-42.331").Root); + Assert.AreEqual(new Ast.DoubleNode(-1.2e19), p.Parse("-1.2e+19").Root); + Assert.AreEqual(new Ast.DoubleNode(-1.2e19), p.Parse("-1.2e19").Root); + Assert.AreEqual(new Ast.DoubleNode(0.0004), p.Parse("4e-4").Root); + Assert.AreEqual(new Ast.DoubleNode(40000), p.Parse("4e4").Root); + Assert.AreEqual(new Ast.BooleanNode(true), p.Parse("True").Root); + Assert.AreEqual(new Ast.BooleanNode(false), p.Parse("False").Root); + Assert.AreEqual(Ast.NoneNode.Instance, p.Parse("None").Root); + + // long ints + Assert.AreEqual(new Ast.DecimalNode(123456789123456789123456789M), p.Parse("123456789123456789123456789").Root); + Assert.AreNotEqual(new Ast.LongNode(52), p.Parse("52").Root); + Assert.AreEqual(new Ast.LongNode(123456789123456789L), p.Parse("123456789123456789").Root); + Assert.Throws(()=>p.Parse("123456789123456789123456789123456789")); // overflow + } + + [Test] + public void TestWeirdFloats() + { + Parser p = new Parser(); + var tuple = (Ast.TupleNode) p.Parse("(1e30000,-1e30000,(1e30000+3.4j),{'__class__':'float','value':'nan'})").Root; + Assert.AreEqual(4, tuple.Elements.Count); + var d = (Ast.DoubleNode) tuple.Elements[0]; + Assert.IsTrue(double.IsPositiveInfinity(d.Value)); + d = (Ast.DoubleNode) tuple.Elements[1]; + Assert.IsTrue(double.IsNegativeInfinity(d.Value)); + var c = (Ast.ComplexNumberNode) tuple.Elements[2]; + Assert.IsTrue(double.IsInfinity(c.Real)); + Assert.AreEqual(3.4, c.Imaginary, 0); + d = (Ast.DoubleNode) tuple.Elements[3]; + Assert.IsTrue(Double.IsNaN(d.Value)); + } + + [Test] + public void TestFloatPrecision() + { + Parser p = new Parser(); + Serializer serpent = new Serializer(); + var ser = serpent.Serialize(1.2345678987654321); + Console.WriteLine(Encoding.UTF8.GetString(ser)); // TODO remove + Ast.DoubleNode dv = (Ast.DoubleNode) p.Parse(ser).Root; + Assert.AreEqual(1.2345678987654321, dv.Value); + + ser = serpent.Serialize(5555.12345678987656); + Console.WriteLine(Encoding.UTF8.GetString(ser)); // TODO remove + dv = (Ast.DoubleNode) p.Parse(ser).Root; + Assert.AreEqual(5555.12345678987656, dv.Value); + + ser = serpent.Serialize(98765432123456.12345678987656); + Console.WriteLine(Encoding.UTF8.GetString(ser)); // TODO remove + dv = (Ast.DoubleNode) p.Parse(ser).Root; + Assert.AreEqual(98765432123456.12345678987656, dv.Value); + + ser = serpent.Serialize(98765432123456.12345678987656e+44); + dv = (Ast.DoubleNode) p.Parse(ser).Root; + Console.WriteLine(Encoding.UTF8.GetString(ser)); // TODO remove + Assert.AreEqual(98765432123456.12345678987656e+44, dv.Value); + + Ast.ComplexNumberNode cv = (Ast.ComplexNumberNode)p.Parse("(98765432123456.12345678987656+665544332211.9998877665544j)").Root; + Assert.AreEqual(98765432123456.12345678987656, cv.Real); + Assert.AreEqual(665544332211.9998877665544, cv.Imaginary); + cv = (Ast.ComplexNumberNode)p.Parse("(98765432123456.12345678987656e+33+665544332211.9998877665544e+44j)").Root; + Assert.AreEqual(98765432123456.12345678987656e+33, cv.Real); + Assert.AreEqual(665544332211.9998877665544e+44, cv.Imaginary); + } + + [Test] + public void TestEquality() + { + Ast.INode n1, n2; + + n1 = new Ast.IntegerNode(42); + n2 = new Ast.IntegerNode(42); + Assert.AreEqual(n1, n2); + n2 = new Ast.IntegerNode(43); + Assert.AreNotEqual(n1, n2); + + n1 = new Ast.StringNode("foo"); + n2 = new Ast.StringNode("foo"); + Assert.AreEqual(n1, n2); + n2 = new Ast.StringNode("bar"); + Assert.AreNotEqual(n1, n2); + + n1 = new Ast.ComplexNumberNode() { + Real=1.1, + Imaginary=2.2 + }; + n2 = new Ast.ComplexNumberNode() { + Real=1.1, + Imaginary=2.2 + }; + Assert.AreEqual(n1, n2); + n2 = new Ast.ComplexNumberNode() { + Real=1.1, + Imaginary=3.3 + }; + Assert.AreNotEqual(n1, n2); + + n1=new Ast.KeyValueNode() { + Key=new Ast.IntegerNode(42), + Value=new Ast.IntegerNode(42) + }; + n2=new Ast.KeyValueNode() { + Key=new Ast.IntegerNode(42), + Value=new Ast.IntegerNode(42) + }; + Assert.AreEqual(n1, n2); + n1=new Ast.KeyValueNode() { + Key=new Ast.IntegerNode(43), + Value=new Ast.IntegerNode(43) + }; + Assert.AreNotEqual(n1,n2); + + n1=Ast.NoneNode.Instance; + n2=Ast.NoneNode.Instance; + Assert.AreEqual(n1, n2); + n2=new Ast.IntegerNode(42); + Assert.AreNotEqual(n1, n2); + + n1=new Ast.DictNode() { + Elements=new List() { + new Ast.KeyValueNode() { + Key=new Ast.IntegerNode(42), + Value=new Ast.IntegerNode(42) + } + } + }; + n2=new Ast.DictNode() { + Elements=new List() { + new Ast.KeyValueNode() { + Key=new Ast.IntegerNode(42), + Value=new Ast.IntegerNode(42) + } + } + }; + Assert.AreEqual(n1, n2); + n2=new Ast.DictNode() { + Elements=new List() { + new Ast.KeyValueNode() { + Key=new Ast.IntegerNode(42), + Value=new Ast.IntegerNode(43) + } + } + }; + Assert.AreNotEqual(n1, n2); + + n1=new Ast.ListNode() { + Elements=new List() { + new Ast.IntegerNode(42) + } + }; + n2=new Ast.ListNode() { + Elements=new List() { + new Ast.IntegerNode(42) + } + }; + Assert.AreEqual(n1,n2); + n2=new Ast.ListNode() { + Elements=new List() { + new Ast.IntegerNode(43) + } + }; + Assert.AreNotEqual(n1,n2); + + n1=new Ast.SetNode() { + Elements=new List() { + new Ast.IntegerNode(42) + } + }; + n2=new Ast.SetNode() { + Elements=new List() { + new Ast.IntegerNode(42) + } + }; + Assert.AreEqual(n1,n2); + n2=new Ast.SetNode() { + Elements=new List() { + new Ast.IntegerNode(43) + } + }; + Assert.AreNotEqual(n1,n2); + + n1=new Ast.TupleNode() { + Elements=new List() { + new Ast.IntegerNode(42) + } + }; + n2=new Ast.TupleNode() { + Elements=new List() { + new Ast.IntegerNode(42) + } + }; + Assert.AreEqual(n1,n2); + n2=new Ast.TupleNode() { + Elements=new List() { + new Ast.IntegerNode(43) + } + }; + Assert.AreNotEqual(n1,n2); + + } + + [Test] + public void TestDictEquality() + { + Ast.DictNode dict1 = new Ast.DictNode(); + Ast.KeyValueNode kv=new Ast.KeyValueNode() { + Key=new Ast.StringNode("key1"), + Value=new Ast.IntegerNode(42) + }; + dict1.Elements.Add(kv); + kv=new Ast.KeyValueNode() { + Key=new Ast.StringNode("key2"), + Value=new Ast.IntegerNode(43) + }; + dict1.Elements.Add(kv); + kv=new Ast.KeyValueNode() { + Key=new Ast.StringNode("key3"), + Value=new Ast.IntegerNode(44) + }; + dict1.Elements.Add(kv); + + Ast.DictNode dict2 = new Ast.DictNode(); + kv=new Ast.KeyValueNode(){ + Key=new Ast.StringNode("key2"), + Value=new Ast.IntegerNode(43) + }; + dict2.Elements.Add(kv); + kv=new Ast.KeyValueNode() { + Key=new Ast.StringNode("key3"), + Value=new Ast.IntegerNode(44) + }; + dict2.Elements.Add(kv); + kv=new Ast.KeyValueNode() { + Key=new Ast.StringNode("key1"), + Value=new Ast.IntegerNode(42) + }; + dict2.Elements.Add(kv); + + Assert.AreEqual(dict1, dict2); + kv=new Ast.KeyValueNode() { + Key=new Ast.StringNode("key4"), + Value=new Ast.IntegerNode(45) + }; + dict2.Elements.Add(kv); + Assert.AreNotEqual(dict1, dict2); + } + + [Test] + public void TestSetEquality() + { + Ast.SetNode set1 = new Ast.SetNode(); + set1.Elements.Add(new Ast.IntegerNode(1)); + set1.Elements.Add(new Ast.IntegerNode(2)); + set1.Elements.Add(new Ast.IntegerNode(3)); + + Ast.SetNode set2 = new Ast.SetNode(); + set2.Elements.Add(new Ast.IntegerNode(2)); + set2.Elements.Add(new Ast.IntegerNode(3)); + set2.Elements.Add(new Ast.IntegerNode(1)); + + Assert.AreEqual(set1, set2); + set2.Elements.Add(new Ast.IntegerNode(0)); + Assert.AreNotEqual(set1, set2); + } + + [Test] + public void TestPrintSingle() + { + Parser p = new Parser(); + + // primitives + Assert.AreEqual("42", p.Parse("42").Root.ToString()); + Assert.AreEqual("-42.331", p.Parse("-42.331").Root.ToString()); + Assert.AreEqual("-42.0", p.Parse("-42.0").Root.ToString()); + Assert.AreEqual("-2E+20", p.Parse("-2E20").Root.ToString()); + Assert.AreEqual("2.0", p.Parse("2.0").Root.ToString()); + Assert.AreEqual("1.2E+19", p.Parse("1.2e19").Root.ToString()); + Assert.AreEqual("True", p.Parse("True").Root.ToString()); + Assert.AreEqual("'hello'", p.Parse("'hello'").Root.ToString()); + Assert.AreEqual("'\\n'", p.Parse("'\n'").Root.ToString()); + Assert.AreEqual("'\\''", p.Parse("'\\''").Root.ToString()); + Assert.AreEqual("'\"'", p.Parse("'\\\"'").Root.ToString()); + Assert.AreEqual("'\"'", p.Parse("'\"'").Root.ToString()); + Assert.AreEqual("'\\\\'", p.Parse("'\\\\'").Root.ToString()); + Assert.AreEqual("None", p.Parse("None").Root.ToString()); + string ustr = "'\u20ac\u2603'"; + Assert.AreEqual(ustr, p.Parse(ustr).Root.ToString()); + + // complex + Assert.AreEqual("(0+2j)", p.Parse("2j").Root.ToString()); + Assert.AreEqual("(-1.1-2.2j)", p.Parse("(-1.1-2.2j)").Root.ToString()); + Assert.AreEqual("(1.1+2.2j)", p.Parse("(1.1+2.2j)").Root.ToString()); + + // long int + Assert.AreEqual("123456789123456789123456789", p.Parse("123456789123456789123456789").Root.ToString()); + } + + [Test] + public void TestPrintSeq() + { + Parser p=new Parser(); + + //tuple + Assert.AreEqual("()", p.Parse("()").Root.ToString()); + Assert.AreEqual("(42,)", p.Parse("(42,)").Root.ToString()); + Assert.AreEqual("(42,43)", p.Parse("(42,43)").Root.ToString()); + + // list + Assert.AreEqual("[]", p.Parse("[]").Root.ToString()); + Assert.AreEqual("[42]", p.Parse("[42]").Root.ToString()); + Assert.AreEqual("[42,43]", p.Parse("[42,43]").Root.ToString()); + + // set + Assert.AreEqual("{42}", p.Parse("{42}").Root.ToString()); + Assert.AreEqual("{42,43}", p.Parse("{42,43,43,43}").Root.ToString()); + + // dict + Assert.AreEqual("{}", p.Parse("{}").Root.ToString()); + Assert.AreEqual("{'a':42}", p.Parse("{'a': 42}").Root.ToString()); + Assert.AreEqual("{'a':42,'b':43}", p.Parse("{'a': 42, 'b': 43}").Root.ToString()); + Assert.AreEqual("{'a':42,'b':45}", p.Parse("{'a': 42, 'b': 43, 'b': 44, 'b': 45}").Root.ToString()); + } + + [Test] + public void TestInvalidPrimitives() + { + Parser p = new Parser(); + Assert.Throws(()=>p.Parse("1+2")); + Assert.Throws(()=>p.Parse("1-2")); + Assert.Throws(()=>p.Parse("1.1+2.2")); + Assert.Throws(()=>p.Parse("1.1-2.2")); + Assert.Throws(()=>p.Parse("True+2")); + Assert.Throws(()=>p.Parse("False-2")); + Assert.Throws(()=>p.Parse("3j+2")); + Assert.Throws(()=>p.Parse("3j-2")); + Assert.Throws(()=>p.Parse("None+2")); + Assert.Throws(()=>p.Parse("None-2")); + } + + [Test] + public void TestComplex() + { + Parser p = new Parser(); + var cplx = new Ast.ComplexNumberNode() { + Real = 4.2, + Imaginary = 3.2 + }; + var cplx2 = new Ast.ComplexNumberNode() { + Real = 4.2, + Imaginary = 99 + }; + Assert.AreNotEqual(cplx, cplx2); + cplx2.Imaginary = 3.2; + Assert.AreEqual(cplx, cplx2); + + Assert.AreEqual(cplx, p.Parse("(4.2+3.2j)").Root); + cplx.Real = 0; + Assert.AreEqual(cplx, p.Parse("(0+3.2j)").Root); + Assert.AreEqual(cplx, p.Parse("3.2j").Root); + Assert.AreEqual(cplx, p.Parse("+3.2j").Root); + cplx.Imaginary = -3.2; + Assert.AreEqual(cplx, p.Parse("-3.2j").Root); + cplx.Real = -9.9; + Assert.AreEqual(cplx, p.Parse("(-9.9-3.2j)").Root); + + cplx.Real = 2; + cplx.Imaginary = 3; + Assert.AreEqual(cplx, p.Parse("(2+3j)").Root); + cplx.Imaginary = -3; + Assert.AreEqual(cplx, p.Parse("(2-3j)").Root); + cplx.Real = 0; + Assert.AreEqual(cplx, p.Parse("-3j").Root); + + cplx.Real = -3.2e32; + cplx.Imaginary = -9.9e44; + Assert.AreEqual(cplx, p.Parse("(-3.2e32 -9.9e44j)").Root); + Assert.AreEqual(cplx, p.Parse("(-3.2e+32 -9.9e+44j)").Root); + Assert.AreEqual(cplx, p.Parse("(-3.2e32-9.9e44j)").Root); + Assert.AreEqual(cplx, p.Parse("(-3.2e+32-9.9e+44j)").Root); + cplx.Imaginary = 9.9e44; + Assert.AreEqual(cplx, p.Parse("(-3.2e32+9.9e44j)").Root); + Assert.AreEqual(cplx, p.Parse("(-3.2e+32+9.9e+44j)").Root); + cplx.Real = -3.2e-32; + cplx.Imaginary = -9.9e-44; + Assert.AreEqual(cplx, p.Parse("(-3.2e-32-9.9e-44j)").Root); + } + + [Test] + public void TestPrimitivesStuffAtEnd() + { + Parser p = new Parser(); + Assert.AreEqual(new Ast.IntegerNode(42), p.ParseSingle(new SeekableStringReader("42@"))); + Assert.AreEqual(new Ast.DoubleNode(42.331), p.ParseSingle(new SeekableStringReader("42.331@"))); + Assert.AreEqual(new Ast.BooleanNode(true), p.ParseSingle(new SeekableStringReader("True@"))); + Assert.AreEqual(Ast.NoneNode.Instance, p.ParseSingle(new SeekableStringReader("None@"))); + var cplx = new Ast.ComplexNumberNode() { + Real = 4, + Imaginary = 3 + }; + Assert.AreEqual(cplx, p.ParseSingle(new SeekableStringReader("(4+3j)@"))); + cplx.Real=0; + Assert.AreEqual(cplx, p.ParseSingle(new SeekableStringReader("3j@"))); + } + + [Test] + public void TestStrings() + { + Parser p = new Parser(); + Assert.AreEqual(new Ast.StringNode("hello"), p.Parse("'hello'").Root); + Assert.AreEqual(new Ast.StringNode("hello"), p.Parse("\"hello\"").Root); + Assert.AreEqual(new Ast.StringNode("\\"), p.Parse("'\\\\'").Root); + Assert.AreEqual(new Ast.StringNode("\\"), p.Parse("\"\\\\\"").Root); + Assert.AreEqual(new Ast.StringNode("'"), p.Parse("\"'\"").Root); + Assert.AreEqual(new Ast.StringNode("\""), p.Parse("'\"'").Root); + Assert.AreEqual(new Ast.StringNode("tab\tnewline\n."), p.Parse("'tab\\tnewline\\n.'").Root); + } + + [Test] + public void TestUnicode() + { + Parser p = new Parser(); + string str = "'\u20ac\u2603'"; + Assert.AreEqual(0x20ac, str[1]); + Assert.AreEqual(0x2603, str[2]); + byte[] bytes = Encoding.UTF8.GetBytes(str); + + string value = "\u20ac\u2603"; + Assert.AreEqual(new Ast.StringNode(value), p.Parse(str).Root); + Assert.AreEqual(new Ast.StringNode(value), p.Parse(bytes).Root); + } + + [Test] + public void TestWhitespace() + { + Parser p = new Parser(); + Assert.AreEqual(new Ast.IntegerNode(42), p.Parse(" 42 ").Root); + Assert.AreEqual(new Ast.IntegerNode(42), p.Parse(" 42 ").Root); + Assert.AreEqual(new Ast.IntegerNode(42), p.Parse("\t42\r\n").Root); + Assert.AreEqual(new Ast.IntegerNode(42), p.Parse(" \t 42 \r \n ").Root); + Assert.AreEqual(new Ast.StringNode(" string value "), p.Parse(" ' string value ' ").Root); + Assert.Throws(()=>p.Parse(" ( 42 , ( 'x', 'y' ) ")); // missing tuple close ) + Ast ast = p.Parse(" ( 42 , ( 'x', 'y' ) ) "); + Ast.TupleNode tuple = (Ast.TupleNode) ast.Root; + Assert.AreEqual(new Ast.IntegerNode(42), tuple.Elements[0]); + tuple = (Ast.TupleNode) tuple.Elements[1]; + Assert.AreEqual(new Ast.StringNode("x"), tuple.Elements[0]); + Assert.AreEqual(new Ast.StringNode("y"), tuple.Elements[1]); + + p.Parse(" ( 52 , ) "); + p.Parse(" [ 52 ] "); + p.Parse(" { 'a' : 42 } "); + p.Parse(" { 52 } "); + } + + [Test] + public void TestTuple() + { + Parser p = new Parser(); + Ast.TupleNode tuple = new Ast.TupleNode(); + Ast.TupleNode tuple2 = new Ast.TupleNode(); + Assert.AreEqual(tuple, tuple2); + + tuple.Elements.Add(new Ast.IntegerNode(42)); + tuple2.Elements.Add(new Ast.IntegerNode(99)); + Assert.AreNotEqual(tuple, tuple2); + tuple2.Elements.Clear(); + tuple2.Elements.Add(new Ast.IntegerNode(42)); + Assert.AreEqual(tuple, tuple2); + tuple2.Elements.Add(new Ast.IntegerNode(43)); + tuple2.Elements.Add(new Ast.IntegerNode(44)); + Assert.AreNotEqual(tuple, tuple2); + + Assert.AreEqual(new Ast.TupleNode(), p.Parse("()").Root); + Assert.AreEqual(tuple, p.Parse("(42,)").Root); + Assert.AreEqual(tuple2, p.Parse("( 42,43, 44 )").Root); + + Assert.Throws(()=>p.Parse("(42,43]")); + Assert.Throws(()=>p.Parse("()@")); + Assert.Throws(()=>p.Parse("(42,43)@")); + } + + [Test] + public void TestList() + { + Parser p = new Parser(); + Ast.ListNode list = new Ast.ListNode(); + Ast.ListNode list2 = new Ast.ListNode(); + Assert.AreEqual(list, list2); + + list.Elements.Add(new Ast.IntegerNode(42)); + list2.Elements.Add(new Ast.IntegerNode(99)); + Assert.AreNotEqual(list, list2); + list2.Elements.Clear(); + list2.Elements.Add(new Ast.IntegerNode(42)); + Assert.AreEqual(list, list2); + list2.Elements.Add(new Ast.IntegerNode(43)); + list2.Elements.Add(new Ast.IntegerNode(44)); + Assert.AreNotEqual(list, list2); + + Assert.AreEqual(new Ast.ListNode(), p.Parse("[]").Root); + Assert.AreEqual(list, p.Parse("[42]").Root); + Assert.AreEqual(list2, p.Parse("[ 42,43, 44 ]").Root); + + Assert.Throws(()=>p.Parse("[42,43}")); + Assert.Throws(()=>p.Parse("[]@")); + Assert.Throws(()=>p.Parse("[42,43]@")); + } + + [Test] + public void TestSet() + { + Parser p = new Parser(); + Ast.SetNode set1 = new Ast.SetNode(); + Ast.SetNode set2 = new Ast.SetNode(); + Assert.AreEqual(set1, set2); + + set1.Elements.Add(new Ast.IntegerNode(42)); + set2.Elements.Add(new Ast.IntegerNode(99)); + Assert.AreNotEqual(set1, set2); + set2.Elements.Clear(); + set2.Elements.Add(new Ast.IntegerNode(42)); + Assert.AreEqual(set1, set2); + + set2.Elements.Add(new Ast.IntegerNode(43)); + set2.Elements.Add(new Ast.IntegerNode(44)); + Assert.AreNotEqual(set1, set2); + + Assert.AreEqual(set1, p.Parse("{42}").Root); + Assert.AreEqual(set2, p.Parse("{ 42,43, 44 }").Root); + + Assert.Throws(()=>p.Parse("{42,43]")); + Assert.Throws(()=>p.Parse("{42,43}@")); + + set1 = p.Parse("{'first','second','third','fourth','fifth','second', 'first', 'third', 'third' }").Root as Ast.SetNode; + Assert.AreEqual("'first'", set1.Elements[0].ToString()); + Assert.AreEqual("'second'", set1.Elements[1].ToString()); + Assert.AreEqual("'third'", set1.Elements[2].ToString()); + Assert.AreEqual("'fourth'", set1.Elements[3].ToString()); + Assert.AreEqual("'fifth'", set1.Elements[4].ToString()); + Assert.AreEqual(5, set1.Elements.Count); + } + + [Test] + public void TestDict() + { + Parser p = new Parser(); + Ast.DictNode dict1 = new Ast.DictNode(); + Ast.DictNode dict2 = new Ast.DictNode(); + Assert.AreEqual(dict1, dict2); + + Ast.KeyValueNode kv1 = new Ast.KeyValueNode { Key=new Ast.StringNode("key"), Value=new Ast.IntegerNode(42) }; + Ast.KeyValueNode kv2 = new Ast.KeyValueNode { Key=new Ast.StringNode("key"), Value=new Ast.IntegerNode(99) }; + Assert.AreNotEqual(kv1, kv2); + kv2.Value = new Ast.IntegerNode(42); + Assert.AreEqual(kv1, kv2); + + dict1.Elements.Add(new Ast.KeyValueNode { Key=new Ast.StringNode("key1"), Value=new Ast.IntegerNode(42) }); + dict2.Elements.Add(new Ast.KeyValueNode { Key=new Ast.StringNode("key1"), Value=new Ast.IntegerNode(99) }); + Assert.AreNotEqual(dict1, dict2); + dict2.Elements.Clear(); + dict2.Elements.Add(new Ast.KeyValueNode { Key=new Ast.StringNode("key1"), Value=new Ast.IntegerNode(42) }); + Assert.AreEqual(dict1, dict2); + + dict2.Elements.Add(new Ast.KeyValueNode { Key=new Ast.StringNode("key2"), Value=new Ast.IntegerNode(43) }); + dict2.Elements.Add(new Ast.KeyValueNode { Key=new Ast.StringNode("key3"), Value=new Ast.IntegerNode(44) }); + Assert.AreNotEqual(dict1, dict2); + + Assert.AreEqual(new Ast.DictNode(), p.Parse("{}").Root); + Assert.AreEqual(dict1, p.Parse("{'key1': 42}").Root); + Assert.AreEqual(dict2, p.Parse("{'key1': 42, 'key2': 43, 'key3':44}").Root); + + Assert.Throws(()=>p.Parse("{'key': 42]")); + Assert.Throws(()=>p.Parse("{}@")); + Assert.Throws(()=>p.Parse("{'key': 42}@")); + + dict1 = p.Parse("{'a': 1, 'b': 2, 'c': 3, 'c': 4, 'c': 5, 'c': 6}").Root as Ast.DictNode; + Assert.AreEqual("'a':1", dict1.Elements[0].ToString()); + Assert.AreEqual("'b':2", dict1.Elements[1].ToString()); + Assert.AreEqual("'c':6", dict1.Elements[2].ToString()); + Assert.AreEqual(3, dict1.Elements.Count); + } + + [Test] + public void TestFile() + { + Parser p = new Parser(); + byte[] ser=File.ReadAllBytes("testserpent.utf8.bin"); + Ast ast = p.Parse(ser); + + string expr = ast.ToString(); + Ast ast2 = p.Parse(expr); + string expr2 = ast2.ToString(); + Assert.AreEqual(expr, expr2); + + StringBuilder sb= new StringBuilder(); + Walk(ast.Root, sb); + string walk1 = sb.ToString(); + sb= new StringBuilder(); + Walk(ast2.Root, sb); + string walk2 = sb.ToString(); + Assert.AreEqual(walk1, walk2); + + // @TODO Assert.AreEqual(ast.Root, ast2.Root); + ast = p.Parse(expr2); + // @TODO Assert.AreEqual(ast.Root, ast2.Root); + } + + [Test] + [Ignore("can't yet get the ast to compare equal on mono")] + public void TestAstEquals() + { + Parser p = new Parser (); + byte[] ser = File.ReadAllBytes ("testserpent.utf8.bin"); + Ast ast = p.Parse(ser); + Ast ast2 = p.Parse(ser); + Assert.AreEqual(ast.Root, ast2.Root); + } + + public void Walk(Ast.INode node, StringBuilder sb) + { + if(node is Ast.SequenceNode) + { + sb.AppendLine(string.Format("{0} (seq)", node.GetType())); + Ast.SequenceNode seq = (Ast.SequenceNode)node; + foreach(Ast.INode child in seq.Elements) { + Walk(child, sb); + } + } + else + sb.AppendLine(string.Format("{0} = {1}", node.GetType(), node.ToString())); + } + + [Test] + public void TestTrailingCommas() + { + Parser p = new Parser(); + Ast.INode result; + result = p.Parse("[1,2,3, ]").Root; + result = p.Parse("[1,2,3 , ]").Root; + result = p.Parse("[1,2,3,]").Root; + Assert.AreEqual("[1,2,3]", result.ToString()); + result = p.Parse("(1,2,3, )").Root; + result = p.Parse("(1,2,3 , )").Root; + result = p.Parse("(1,2,3,)").Root; + Assert.AreEqual("(1,2,3)", result.ToString()); + + // for dict and set the asserts are a bit more complex + // we cannot simply convert to string because the order of elts is undefined. + + result = p.Parse("{'a':1, 'b':2, 'c':3, }").Root; + result = p.Parse("{'a':1, 'b':2, 'c':3 , }").Root; + result = p.Parse("{'a':1, 'b':2, 'c':3,}").Root; + Ast.DictNode dict = (Ast.DictNode) result; + var items = dict.ElementsAsSet(); + Assert.IsTrue(items.Contains(new Ast.KeyValueNode(new Ast.StringNode("a"), new Ast.IntegerNode(1)))); + Assert.IsTrue(items.Contains(new Ast.KeyValueNode(new Ast.StringNode("b"), new Ast.IntegerNode(2)))); + Assert.IsTrue(items.Contains(new Ast.KeyValueNode(new Ast.StringNode("c"), new Ast.IntegerNode(3)))); + result = p.Parse("{1,2,3, }").Root; + result = p.Parse("{1,2,3 , }").Root; + result = p.Parse("{1,2,3,}").Root; + Ast.SetNode set = (Ast.SetNode) result; + items = set.ElementsAsSet(); + Assert.IsTrue(items.Contains(new Ast.IntegerNode(1))); + Assert.IsTrue(items.Contains(new Ast.IntegerNode(2))); + Assert.IsTrue(items.Contains(new Ast.IntegerNode(3))); + Assert.IsFalse(items.Contains(new Ast.IntegerNode(4))); + } + } + + [TestFixture] + public class VisitorTest + { + [Test] + public void TestObjectify() + { + Parser p = new Parser(); + byte[] ser=File.ReadAllBytes("testserpent.utf8.bin"); + Ast ast = p.Parse(ser); + var visitor = new ObjectifyVisitor(); + ast.Accept(visitor); + object thing = visitor.GetObject(); + + IDictionary dict = (IDictionary) thing; + Assert.AreEqual(11, dict.Count); + IList list = dict["numbers"] as IList; + Assert.AreEqual(4, list.Count); + Assert.AreEqual(999.1234, list[1]); + Assert.AreEqual(new ComplexNumber(-3, 8), list[3]); + string euro = dict["unicode"] as string; + Assert.AreEqual("\u20ac", euro); + IDictionary exc = (IDictionary)dict["exc"]; + object[] args = (object[]) exc["args"]; + Assert.AreEqual("fault", args[0]); + Assert.AreEqual("ZeroDivisionError", exc["__class__"]); + } + + object ZerodivisionFromDict(IDictionary dict) + { + string classname = (string)dict["__class__"]; + if(classname=="ZeroDivisionError") + { + object[] args = (object[]) dict["args"]; + return new DivideByZeroException((string)args[0]); + } + return null; + } + + [Test] + public void TestObjectifyDictToClass() + { + Parser p = new Parser(); + byte[] ser=File.ReadAllBytes("testserpent.utf8.bin"); + Ast ast = p.Parse(ser); + + var visitor = new ObjectifyVisitor(ZerodivisionFromDict); + ast.Accept(visitor); + object thing = visitor.GetObject(); + + IDictionary dict = (IDictionary) thing; + Assert.AreEqual(11, dict.Count); + DivideByZeroException ex = (DivideByZeroException) dict["exc"]; + Assert.AreEqual("fault", ex.Message); + + thing = ast.GetData(ZerodivisionFromDict); + dict = (IDictionary) thing; + Assert.AreEqual(11, dict.Count); + ex = (DivideByZeroException) dict["exc"]; + Assert.AreEqual("fault", ex.Message); + } + } } \ No newline at end of file diff --git a/dotnet/Serpent.Test/SerializeTest.cs b/dotnet/Serpent.Test/SerializeTest.cs index 4758499..bfe2c1a 100644 --- a/dotnet/Serpent.Test/SerializeTest.cs +++ b/dotnet/Serpent.Test/SerializeTest.cs @@ -1,701 +1,701 @@ -/// -/// Serpent, a Python literal expression serializer/deserializer -/// (a.k.a. Python's ast.literal_eval in .NET) -/// -/// Copyright Irmen de Jong (irmen@razorvine.net) -/// Software license: "MIT software license". See http://opensource.org/licenses/MIT -/// - -using System; -using System.Collections.Generic; -using System.Text; - -using NUnit.Framework; -using Hashtable = System.Collections.Hashtable; -using IDictionary = System.Collections.IDictionary; - -namespace Razorvine.Serpent.Test -{ - [TestFixture] - public class SerializeTest - { - public byte[] strip_header(byte[] data) - { - int start=Array.IndexOf(data, (byte)10); // the newline after the header - if(start<0) - throw new ArgumentException("need header in string"); - start++; - byte[] result = new byte[data.Length-start]; - Array.Copy(data, start, result, 0, data.Length-start); - return result; - } - - public byte[] B(string s) - { - return Encoding.UTF8.GetBytes(s); - } - - public string S(byte[] b) - { - return Encoding.UTF8.GetString(b); - } - - - [Test] - public void TestHeader() - { - Serializer ser = new Serializer(); - byte[] data = ser.Serialize(null); - Assert.AreEqual(35, data[0]); - string strdata = S(data); - Assert.AreEqual("# serpent utf-8 python3.2", strdata.Split('\n')[0]); - - ser.SetLiterals=false; - data = ser.Serialize(null); - strdata = S(data); - Assert.AreEqual("# serpent utf-8 python2.6", strdata.Split('\n')[0]); - - data = B("# header\nfirst-line"); - data = strip_header(data); - Assert.AreEqual(B("first-line"), data); - } - - - [Test] - public void TestStuff() - { - Serializer ser=new Serializer(); - byte[] result = ser.Serialize("blerp"); - result=strip_header(result); - Assert.AreEqual(B("'blerp'"), result); - result = ser.Serialize(new Guid("f1f8d00e-49a5-4662-ac1d-d5f0426ed293")); - result=strip_header(result); - Assert.AreEqual(B("'f1f8d00e-49a5-4662-ac1d-d5f0426ed293'"), result); - result = ser.Serialize(123456789.987654321987654321987654321987654321m); - result=strip_header(result); - Assert.AreEqual(B("'123456789.98765432198765432199'"), result); - } - - [Test] - public void TestNull() - { - Serializer ser = new Serializer(); - byte[] data = ser.Serialize(null); - data=strip_header(data); - Assert.AreEqual(B("None"),data); - } - - [Test] - public void TestStrings() - { - Serializer serpent = new Serializer(); - byte[] ser = serpent.Serialize("hello"); - byte[] data = strip_header(ser); - Assert.AreEqual(B("'hello'"), data); - ser = serpent.Serialize("quotes'\""); - data = strip_header(ser); - Assert.AreEqual(B("'quotes\\'\"'"), data); - ser = serpent.Serialize("quotes2'"); - data = strip_header(ser); - Assert.AreEqual(B("\"quotes2'\""), data); - } - - [Test] - public void TestUnicode() - { - Serializer serpent = new Serializer(); - byte[] ser = serpent.Serialize("euro\u20ac"); - byte[] data = strip_header(ser); - Assert.AreEqual(new byte[] {39, 101, 117, 114, 111, 0xe2, 0x82, 0xac, 39}, data); - - ser = serpent.Serialize("A\n\t\\Z"); - // 'A\\n\\t\\\\Z' (10 bytes) - data = strip_header(ser); - Assert.AreEqual(new byte[] {39, 65, 92, 110, 92, 116, 92, 92, 90, 39}, data); - - ser = serpent.Serialize("euro\u20ac\nlastline\ttab\\@slash"); - // 'euro\xe2\x82\xac\\nlastline\\ttab\\\\@slash' (32 bytes) - data = strip_header(ser); - Assert.AreEqual(new byte[] { - 39, 101, 117, 114, 111, 226, 130, 172, - 92, 110, 108, 97, 115, 116, 108, 105, - 110, 101, 92, 116, 116, 97, 98, 92, - 92, 64, 115, 108, 97, 115, 104, 39} - , data); - } - - [Test] - public void TestNumbers() - { - Serializer serpent = new Serializer(); - byte[] ser = serpent.Serialize((int)12345); - byte[] data = strip_header(ser); - Assert.AreEqual(B("12345"), data); - ser = serpent.Serialize((uint)12345); - data = strip_header(ser); - Assert.AreEqual(B("12345"), data); - ser = serpent.Serialize((long)1234567891234567891L); - data = strip_header(ser); - Assert.AreEqual(B("1234567891234567891"), data); - ser = serpent.Serialize((ulong)12345678912345678912L); - data = strip_header(ser); - Assert.AreEqual(B("12345678912345678912"), data); - ser = serpent.Serialize(99.1234); - data = strip_header(ser); - Assert.AreEqual(B("99.1234"), data); - ser = serpent.Serialize(1234.9999999999m); - data = strip_header(ser); - Assert.AreEqual(B("'1234.9999999999'"), data); - ser = serpent.Serialize(123456789.987654321987654321987654321987654321m); - data=strip_header(ser); - Assert.AreEqual(B("'123456789.98765432198765432199'"), data); - ComplexNumber cplx = new ComplexNumber(2.2, 3.3); - ser = serpent.Serialize(cplx); - data = strip_header(ser); - Assert.AreEqual(B("(2.2+3.3j)"), data); - cplx = new ComplexNumber(0, 3); - ser = serpent.Serialize(cplx); - data = strip_header(ser); - Assert.AreEqual(B("(0+3j)"), data); - cplx = new ComplexNumber(-2, -3); - ser = serpent.Serialize(cplx); - data = strip_header(ser); - Assert.AreEqual(B("(-2-3j)"), data); - } - - [Test] - public void testDoubleNanInf() - { - Serializer serpent = new Serializer(); - var doubles = new object[] {double.PositiveInfinity, double.NegativeInfinity, double.NaN, - float.PositiveInfinity, float.NegativeInfinity, float.NaN, - new ComplexNumber(double.PositiveInfinity, 3.4)}; - byte[] ser = serpent.Serialize(doubles); - byte[] data = strip_header(ser); - Assert.AreEqual("(1e30000,-1e30000,{'__class__':'float','value':'nan'},1e30000,-1e30000,{'__class__':'float','value':'nan'},(1e30000+3.4j))", S(data)); - } - - [Test] - public void TestBool() - { - Serializer serpent = new Serializer(); - byte[] ser = serpent.Serialize(true); - byte[] data = strip_header(ser); - Assert.AreEqual(B("True"),data); - ser = serpent.Serialize(false); - data = strip_header(ser); - Assert.AreEqual(B("False"),data); - } - - [Test] - public void TestList() - { - Serializer serpent = new Serializer(); - IList list = new List(); - - // test empty list - byte[] ser = strip_header(serpent.Serialize(list)); - Assert.AreEqual("[]", S(ser)); - serpent.Indent=true; - ser = strip_header(serpent.Serialize(list)); - Assert.AreEqual("[]", S(ser)); - serpent.Indent=false; - - // test nonempty list - list.Add(42); - list.Add("Sally"); - list.Add(16.5); - ser = strip_header(serpent.Serialize(list)); - Assert.AreEqual("[42,'Sally',16.5]", S(ser)); - serpent.Indent=true; - ser = strip_header(serpent.Serialize(list)); - Assert.AreEqual("[\n 42,\n 'Sally',\n 16.5\n]", S(ser)); - } - - [Test] - public void TestSet() - { - // test with set literals - Serializer serpent = new Serializer(); - serpent.SetLiterals = true; - HashSet set = new HashSet(); - - // test empty set - byte[] ser = strip_header(serpent.Serialize(set)); - Assert.AreEqual("()", S(ser)); // empty set is serialized as a tuple. - serpent.Indent=true; - ser = strip_header(serpent.Serialize(set)); - Assert.AreEqual("()", S(ser)); // empty set is serialized as a tuple. - serpent.Indent=false; - - // test nonempty set - set.Add(42); - set.Add("Sally"); - set.Add(16.5); - ser = strip_header(serpent.Serialize(set)); - Assert.AreEqual("{42,'Sally',16.5}", S(ser)); - serpent.Indent=true; - ser = strip_header(serpent.Serialize(set)); - Assert.AreEqual("{\n 42,\n 'Sally',\n 16.5\n}", S(ser)); - - // test no set literals - serpent.Indent=false; - serpent.SetLiterals=false; - ser = strip_header(serpent.Serialize(set)); - Assert.AreEqual("(42,'Sally',16.5)", S(ser)); // needs to be tuple now - } - - [Test] - public void TestDictionary() - { - Serializer serpent = new Serializer(); - Parser p = new Parser(); - - // test empty dict - IDictionary ht = new Hashtable(); - byte[] ser = serpent.Serialize(ht); - Assert.AreEqual(B("{}"), strip_header(ser)); - string parsed = p.Parse(ser).Root.ToString(); - Assert.AreEqual("{}", parsed); - - // empty dict with indentation - serpent.Indent=true; - ser = serpent.Serialize(ht); - Assert.AreEqual(B("{}"), strip_header(ser)); - parsed = p.Parse(ser).Root.ToString(); - Assert.AreEqual("{}", parsed); - - // test dict with values - serpent.Indent=false; - ht = new Hashtable() { - {42, "fortytwo"}, - {"sixteen-and-half", 16.5}, - {"name", "Sally"}, - {"status", false} - }; - - ser = serpent.Serialize(ht); - Assert.AreEqual('}', ser[ser.Length-1]); - Assert.AreNotEqual(',', ser[ser.Length-2]); - parsed = p.Parse(ser).Root.ToString(); - Assert.AreEqual(69, parsed.Length); - - // test indentation - serpent.Indent=true; - ser = serpent.Serialize(ht); - Assert.AreEqual('}', ser[ser.Length-1]); - Assert.AreEqual('\n', ser[ser.Length-2]); - Assert.AreNotEqual(',', ser[ser.Length-3]); - string ser_str = S(strip_header(ser)); - Assert.IsTrue(ser_str.Contains("'name': 'Sally'")); - Assert.IsTrue(ser_str.Contains("'status': False")); - Assert.IsTrue(ser_str.Contains("42: 'fortytwo'")); - Assert.IsTrue(ser_str.Contains("'sixteen-and-half': 16.5")); - parsed = p.Parse(ser).Root.ToString(); - Assert.AreEqual(69, parsed.Length); - serpent.Indent=false; - - // generic Dictionary test - IDictionary mydict = new Dictionary { - { 1, "one" }, - { 2, "two" }, - }; - ser = serpent.Serialize(mydict); - ser_str = S(strip_header(ser)); - Assert.IsTrue(ser_str=="{2:'two',1:'one'}" || ser_str=="{1:'one',2:'two'}"); - } - - [Test] - public void TestBytes() - { - Serializer serpent = new Serializer(indent: true); - byte[] bytes = new byte[] { 97, 98, 99, 100, 101, 102 }; // abcdef - byte[] ser = serpent.Serialize(bytes); - Assert.AreEqual("{\n 'data': 'YWJjZGVm',\n 'encoding': 'base64'\n}", S(strip_header(ser))); - - Parser p = new Parser(); - string parsed = p.Parse(ser).Root.ToString(); - Assert.AreEqual(39, parsed.Length); - - var hashtable = new Hashtable { - {"data", "YWJjZGVm"}, - {"encoding", "base64"} - }; - byte[] bytes2 = Parser.ToBytes(hashtable); - Assert.AreEqual(bytes, bytes2); - - var dict = new Dictionary { - {"data", "YWJjZGVm"}, - {"encoding", "base64"} - }; - bytes2 = Parser.ToBytes(dict); - Assert.AreEqual(bytes, bytes2); - - var dict2 = new Dictionary { - {"data", "YWJjZGVm"}, - {"encoding", "base64"} - }; - bytes2 = Parser.ToBytes(dict2); - Assert.AreEqual(bytes, bytes2); - - dict["encoding"] = "base99"; - Assert.Throws(()=>Parser.ToBytes(dict)); - dict.Clear(); - Assert.Throws(()=>Parser.ToBytes(dict)); - dict.Clear(); - dict["data"] = "YWJjZGVm"; - Assert.Throws(()=>Parser.ToBytes(dict)); - dict.Clear(); - dict["encoding"] = "base64"; - Assert.Throws(()=>Parser.ToBytes(dict)); - Assert.Throws(()=>Parser.ToBytes(12345)); - Assert.Throws(()=>Parser.ToBytes(null)); - } - - [Test] - public void TestCollection() - { - ICollection intlist = new LinkedList(); - intlist.Add(42); - intlist.Add(43); - Serializer serpent = new Serializer(); - byte[] ser = serpent.Serialize(intlist); - ser = strip_header(ser); - Assert.AreEqual("[42,43]", S(ser)); - - ser=strip_header(serpent.Serialize(new int[] {42})); - Assert.AreEqual("(42,)", S(ser)); - ser=strip_header(serpent.Serialize(new int[] {42, 43})); - Assert.AreEqual("(42,43)", S(ser)); - - serpent.Indent=true; - ser = strip_header(serpent.Serialize(intlist)); - Assert.AreEqual("[\n 42,\n 43\n]", S(ser)); - ser=strip_header(serpent.Serialize(new int[] {42})); - Assert.AreEqual("(\n 42,\n)", S(ser)); - ser=strip_header(serpent.Serialize(new int[] {42, 43})); - Assert.AreEqual("(\n 42,\n 43\n)", S(ser)); - } - - - [Test] - public void TestIndentation() - { - var dict = new Dictionary(); - var list = new List() { - 1, - 2, - new string[] {"a", "b"} - }; - dict.Add("first", list); - dict.Add("second", new Dictionary { - {1, false} - }); - dict.Add("third", new HashSet { 3, 4} ); - - Serializer serpent = new Serializer(); - serpent.Indent=true; - byte[] ser = strip_header(serpent.Serialize(dict)); - string txt=@"{ - 'first': [ - 1, - 2, - ( - 'a', - 'b' - ) - ], - 'second': { - 1: False - }, - 'third': { - 3, - 4 - } -}"; - // bit of trickery to deal with Windows/Unix line ending differences - txt = txt.Replace("\n","\r\n"); - txt = txt.Replace("\r\r\n", "\r\n"); - string ser_txt = S(ser); - ser_txt = ser_txt.Replace("\n", "\r\n"); - ser_txt = ser_txt.Replace("\r\r\n", "\r\n"); - Assert.AreEqual(txt, ser_txt); - } - - [Test] - public void TestSorting() - { - Serializer serpent=new Serializer(); - object data = new List { 3, 2, 1}; - byte[] ser = strip_header(serpent.Serialize(data)); - Assert.AreEqual("[3,2,1]", S(ser)); - data = new int[] { 3,2,1 }; - ser = strip_header(serpent.Serialize(data)); - Assert.AreEqual("(3,2,1)", S(ser)); - - data = new HashSet { - 42, - "hi" - }; - serpent.Indent=true; - ser = strip_header(serpent.Serialize(data)); - Assert.IsTrue(S(ser)=="{\n 42,\n 'hi'\n}" || S(ser)=="{\n 'hi',\n 42\n}"); - - data = new Dictionary { - {5, "five"}, - {3, "three"}, - {1, "one"}, - {4, "four"}, - {2, "two"} - }; - serpent.Indent=true; - ser = strip_header(serpent.Serialize(data)); - Assert.AreEqual("{\n 1: 'one',\n 2: 'two',\n 3: 'three',\n 4: 'four',\n 5: 'five'\n}", S(ser)); - - data = new HashSet { - "x", - "y", - "z", - "c", - "b", - "a" - }; - serpent.Indent=true; - ser = strip_header(serpent.Serialize(data)); - Assert.AreEqual("{\n 'a',\n 'b',\n 'c',\n 'x',\n 'y',\n 'z'\n}", S(ser)); - } - - [Test] - public void TestClass() - { - Serializer.RegisterClass(typeof(SerializeTestClass), null); - Serializer serpent = new Serializer(indent: true); - - var obj = new SerializeTestClass() { - i = 99, - s = "hi", - x = 42 - }; - byte[] ser = strip_header(serpent.Serialize(obj)); - Assert.AreEqual("{\n '__class__': 'SerializeTestClass',\n 'i': 99,\n 'obj': None,\n 's': 'hi'\n}", S(ser)); - } - - [Test] - public void TestClass2() - { - Serializer.RegisterClass(typeof(SerializeTestClass), null); - Serializer serpent = new Serializer(indent: true, namespaceInClassName: true); - object obj = new SerializeTestClass() { - i = 99, - s = "hi", - x = 42 - }; - byte[] ser = strip_header(serpent.Serialize(obj)); - Assert.AreEqual("{\n '__class__': 'Razorvine.Serpent.Test.SerializeTestClass',\n 'i': 99,\n 'obj': None,\n 's': 'hi'\n}", S(ser)); - } - - protected IDictionary testclassConverter(object obj) - { - SerializeTestClass o = (SerializeTestClass) obj; - IDictionary result = new Hashtable(); - result["__class@__"] = o.GetType().Name+"@"; - result["i@"] = o.i; - result["s@"] = o.s; - result["x@"] = o.x; - return result; - } - - [Test] - public void TestCustomClassDict() - { - Serializer.RegisterClass(typeof(SerializeTestClass), testclassConverter); - Serializer serpent = new Serializer(indent: true); - - var obj = new SerializeTestClass() { - i = 99, - s = "hi", - x = 42 - }; - byte[] ser = strip_header(serpent.Serialize(obj)); - Assert.AreEqual("{\n '__class@__': 'SerializeTestClass@',\n 'i@': 99,\n 's@': 'hi',\n 'x@': 42\n}", S(ser)); - } - - [Test] - public void TestStruct() - { - Serializer serpent = new Serializer(indent: true); - - var obj2 = new SerializeTestStruct() { - i = 99, - s = "hi", - x = 42 - }; - byte[] ser = strip_header(serpent.Serialize(obj2)); - Assert.AreEqual("{\n '__class__': 'SerializeTestStruct',\n 'i': 99,\n 's': 'hi'\n}", S(ser)); - } - - [Test] - public void TestStruct2() - { - Serializer serpent = new Serializer(indent: true, namespaceInClassName: true); - - var obj2 = new SerializeTestStruct() { - i = 99, - s = "hi", - x = 42 - }; - byte[] ser = strip_header(serpent.Serialize(obj2)); - Assert.AreEqual("{\n '__class__': 'Razorvine.Serpent.Test.SerializeTestStruct',\n 'i': 99,\n 's': 'hi'\n}", S(ser)); - } - - [Test] - public void TestAnonymousClass() - { - Serializer serpent = new Serializer(indent: true); - Object obj = new { - Name="Harry", - Age=33, - Country="NL" - }; - - byte[] ser = strip_header(serpent.Serialize(obj)); - Assert.AreEqual("{\n 'Age': 33,\n 'Country': 'NL',\n 'Name': 'Harry'\n}", S(ser)); - } - - [Test] - public void TestDateTime() - { - Serializer serpent = new Serializer(); - - DateTime date = new DateTime(2013, 1, 20, 23, 59, 45, 999, DateTimeKind.Local); - byte[] ser = strip_header(serpent.Serialize(date)); - Assert.AreEqual("'2013-01-20T23:59:45.999'", S(ser)); - - date = new DateTime(2013, 1, 20, 23, 59, 45, 999, DateTimeKind.Utc); - ser = strip_header(serpent.Serialize(date)); - Assert.AreEqual("'2013-01-20T23:59:45.999'", S(ser)); - - date = new DateTime(2013, 1, 20, 23, 59, 45, 999, DateTimeKind.Unspecified); - ser = strip_header(serpent.Serialize(date)); - Assert.AreEqual("'2013-01-20T23:59:45.999'", S(ser)); - - date = new DateTime(2013, 1, 20, 23, 59, 45); - ser = strip_header(serpent.Serialize(date)); - Assert.AreEqual("'2013-01-20T23:59:45'", S(ser)); - - TimeSpan timespan = new TimeSpan(1, 10, 20, 30, 999); - ser = strip_header(serpent.Serialize(timespan)); - Assert.AreEqual("123630.999", S(ser)); - } - - [Test] - public void TestDateTimeOffset() - { - Serializer serpent = new Serializer(); - - DateTimeOffset date = new DateTimeOffset(2013, 1, 20, 23, 59, 45, 999, TimeSpan.FromHours(+2)); - byte[] ser = strip_header(serpent.Serialize(date)); - Assert.AreEqual("'2013-01-20T23:59:45.999+02:00'", S(ser)); - - date = new DateTimeOffset(2013, 5, 10, 13, 59, 45, TimeSpan.FromHours(+2)); - ser = strip_header(serpent.Serialize(date)); - Assert.AreEqual("'2013-05-10T13:59:45+02:00'", S(ser)); - } - - [Test] - public void TestException() - { - Exception x = new ApplicationException("errormessage"); - Serializer serpent = new Serializer(indent:true); - byte[] ser = strip_header(serpent.Serialize(x)); - Assert.AreEqual("{\n '__class__': 'ApplicationException',\n '__exception__': True,\n 'args': (\n 'errormessage',\n ),\n 'attributes': {}\n}", S(ser)); - - x.Data["custom_attribute"]=999; - ser = strip_header(serpent.Serialize(x)); - Assert.AreEqual("{\n '__class__': 'ApplicationException',\n '__exception__': True,\n 'args': (\n 'errormessage',\n ),\n 'attributes': {\n 'custom_attribute': 999\n }\n}", S(ser)); - } - - [Test] - public void TestExceptionWithNamespace() - { - Exception x = new ApplicationException("errormessage"); - Serializer serpent = new Serializer(indent:true, namespaceInClassName: true); - byte[] ser = strip_header(serpent.Serialize(x)); - Assert.AreEqual("{\n '__class__': 'System.ApplicationException',\n '__exception__': True,\n 'args': (\n 'errormessage',\n ),\n 'attributes': {}\n}", S(ser)); - } - - enum FooType { - Foobar, - Jarjar - } - - [Test] - public void TestEnum() - { - FooType e = FooType.Jarjar; - Serializer serpent = new Serializer(); - byte[] ser = strip_header(serpent.Serialize(e)); - Assert.AreEqual("'Jarjar'", S(ser)); - } - - - interface IBaseInterface {}; - interface ISubInterface : IBaseInterface {}; - class BaseClassWithInterface : IBaseInterface {}; - class SubClassWithInterface : BaseClassWithInterface, ISubInterface {}; - class BaseClass {}; - class SubClass : BaseClass {}; - abstract class AbstractBaseClass {}; - class ConcreteSubClass : AbstractBaseClass {}; - - protected IDictionary AnyClassSerializer(object arg) - { - IDictionary result = new Hashtable(); - result["(SUB)CLASS"] = arg.GetType().Name; - return result; - } - - [Test] - public void testAbstractBaseClassHierarchyPickler() - { - ConcreteSubClass c = new ConcreteSubClass(); - Serializer serpent = new Serializer(); - serpent.Serialize(c); - - Serializer.RegisterClass(typeof(AbstractBaseClass), AnyClassSerializer); - byte[] data = serpent.Serialize(c); - Assert.AreEqual("{'(SUB)CLASS':'ConcreteSubClass'}", S(strip_header(data))); - } - - [Test] - public void testInterfaceHierarchyPickler() - { - BaseClassWithInterface b = new BaseClassWithInterface(); - SubClassWithInterface sub = new SubClassWithInterface(); - Serializer serpent = new Serializer(); - serpent.Serialize(b); - serpent.Serialize(sub); - Serializer.RegisterClass(typeof(IBaseInterface), AnyClassSerializer); - byte[] data = serpent.Serialize(b); - Assert.AreEqual("{'(SUB)CLASS':'BaseClassWithInterface'}", S(strip_header(data))); - data = serpent.Serialize(sub); - Assert.AreEqual("{'(SUB)CLASS':'SubClassWithInterface'}", S(strip_header(data))); - } - } - - public class SerializeTestClass - { - public int x; - public string s {get; set;} - public int i {get; set;} - public object obj {get; set;} - - } - - public struct SerializeTestStruct - { - public int x; - public string s {get; set;} - public int i {get; set;} - } -} +/// +/// Serpent, a Python literal expression serializer/deserializer +/// (a.k.a. Python's ast.literal_eval in .NET) +/// +/// Copyright Irmen de Jong (irmen@razorvine.net) +/// Software license: "MIT software license". See http://opensource.org/licenses/MIT +/// + +using System; +using System.Collections.Generic; +using System.Text; + +using NUnit.Framework; +using Hashtable = System.Collections.Hashtable; +using IDictionary = System.Collections.IDictionary; + +namespace Razorvine.Serpent.Test +{ + [TestFixture] + public class SerializeTest + { + public byte[] strip_header(byte[] data) + { + int start=Array.IndexOf(data, (byte)10); // the newline after the header + if(start<0) + throw new ArgumentException("need header in string"); + start++; + byte[] result = new byte[data.Length-start]; + Array.Copy(data, start, result, 0, data.Length-start); + return result; + } + + public byte[] B(string s) + { + return Encoding.UTF8.GetBytes(s); + } + + public string S(byte[] b) + { + return Encoding.UTF8.GetString(b); + } + + + [Test] + public void TestHeader() + { + Serializer ser = new Serializer(); + byte[] data = ser.Serialize(null); + Assert.AreEqual(35, data[0]); + string strdata = S(data); + Assert.AreEqual("# serpent utf-8 python3.2", strdata.Split('\n')[0]); + + ser.SetLiterals=false; + data = ser.Serialize(null); + strdata = S(data); + Assert.AreEqual("# serpent utf-8 python2.6", strdata.Split('\n')[0]); + + data = B("# header\nfirst-line"); + data = strip_header(data); + Assert.AreEqual(B("first-line"), data); + } + + + [Test] + public void TestStuff() + { + Serializer ser=new Serializer(); + byte[] result = ser.Serialize("blerp"); + result=strip_header(result); + Assert.AreEqual(B("'blerp'"), result); + result = ser.Serialize(new Guid("f1f8d00e-49a5-4662-ac1d-d5f0426ed293")); + result=strip_header(result); + Assert.AreEqual(B("'f1f8d00e-49a5-4662-ac1d-d5f0426ed293'"), result); + result = ser.Serialize(123456789.987654321987654321987654321987654321m); + result=strip_header(result); + Assert.AreEqual(B("'123456789.98765432198765432199'"), result); + } + + [Test] + public void TestNull() + { + Serializer ser = new Serializer(); + byte[] data = ser.Serialize(null); + data=strip_header(data); + Assert.AreEqual(B("None"),data); + } + + [Test] + public void TestStrings() + { + Serializer serpent = new Serializer(); + byte[] ser = serpent.Serialize("hello"); + byte[] data = strip_header(ser); + Assert.AreEqual(B("'hello'"), data); + ser = serpent.Serialize("quotes'\""); + data = strip_header(ser); + Assert.AreEqual(B("'quotes\\'\"'"), data); + ser = serpent.Serialize("quotes2'"); + data = strip_header(ser); + Assert.AreEqual(B("\"quotes2'\""), data); + } + + [Test] + public void TestUnicode() + { + Serializer serpent = new Serializer(); + byte[] ser = serpent.Serialize("euro\u20ac"); + byte[] data = strip_header(ser); + Assert.AreEqual(new byte[] {39, 101, 117, 114, 111, 0xe2, 0x82, 0xac, 39}, data); + + ser = serpent.Serialize("A\n\t\\Z"); + // 'A\\n\\t\\\\Z' (10 bytes) + data = strip_header(ser); + Assert.AreEqual(new byte[] {39, 65, 92, 110, 92, 116, 92, 92, 90, 39}, data); + + ser = serpent.Serialize("euro\u20ac\nlastline\ttab\\@slash"); + // 'euro\xe2\x82\xac\\nlastline\\ttab\\\\@slash' (32 bytes) + data = strip_header(ser); + Assert.AreEqual(new byte[] { + 39, 101, 117, 114, 111, 226, 130, 172, + 92, 110, 108, 97, 115, 116, 108, 105, + 110, 101, 92, 116, 116, 97, 98, 92, + 92, 64, 115, 108, 97, 115, 104, 39} + , data); + } + + [Test] + public void TestNumbers() + { + Serializer serpent = new Serializer(); + byte[] ser = serpent.Serialize((int)12345); + byte[] data = strip_header(ser); + Assert.AreEqual(B("12345"), data); + ser = serpent.Serialize((uint)12345); + data = strip_header(ser); + Assert.AreEqual(B("12345"), data); + ser = serpent.Serialize((long)1234567891234567891L); + data = strip_header(ser); + Assert.AreEqual(B("1234567891234567891"), data); + ser = serpent.Serialize((ulong)12345678912345678912L); + data = strip_header(ser); + Assert.AreEqual(B("12345678912345678912"), data); + ser = serpent.Serialize(99.1234); + data = strip_header(ser); + Assert.AreEqual(B("99.1234"), data); + ser = serpent.Serialize(1234.9999999999m); + data = strip_header(ser); + Assert.AreEqual(B("'1234.9999999999'"), data); + ser = serpent.Serialize(123456789.987654321987654321987654321987654321m); + data=strip_header(ser); + Assert.AreEqual(B("'123456789.98765432198765432199'"), data); + ComplexNumber cplx = new ComplexNumber(2.2, 3.3); + ser = serpent.Serialize(cplx); + data = strip_header(ser); + Assert.AreEqual(B("(2.2+3.3j)"), data); + cplx = new ComplexNumber(0, 3); + ser = serpent.Serialize(cplx); + data = strip_header(ser); + Assert.AreEqual(B("(0+3j)"), data); + cplx = new ComplexNumber(-2, -3); + ser = serpent.Serialize(cplx); + data = strip_header(ser); + Assert.AreEqual(B("(-2-3j)"), data); + } + + [Test] + public void TestDoubleNanInf() + { + Serializer serpent = new Serializer(); + var doubles = new object[] {double.PositiveInfinity, double.NegativeInfinity, double.NaN, + float.PositiveInfinity, float.NegativeInfinity, float.NaN, + new ComplexNumber(double.PositiveInfinity, 3.4)}; + byte[] ser = serpent.Serialize(doubles); + byte[] data = strip_header(ser); + Assert.AreEqual("(1e30000,-1e30000,{'__class__':'float','value':'nan'},1e30000,-1e30000,{'__class__':'float','value':'nan'},(1e30000+3.4j))", S(data)); + } + + [Test] + public void TestBool() + { + Serializer serpent = new Serializer(); + byte[] ser = serpent.Serialize(true); + byte[] data = strip_header(ser); + Assert.AreEqual(B("True"),data); + ser = serpent.Serialize(false); + data = strip_header(ser); + Assert.AreEqual(B("False"),data); + } + + [Test] + public void TestList() + { + Serializer serpent = new Serializer(); + IList list = new List(); + + // test empty list + byte[] ser = strip_header(serpent.Serialize(list)); + Assert.AreEqual("[]", S(ser)); + serpent.Indent=true; + ser = strip_header(serpent.Serialize(list)); + Assert.AreEqual("[]", S(ser)); + serpent.Indent=false; + + // test nonempty list + list.Add(42); + list.Add("Sally"); + list.Add(16.5); + ser = strip_header(serpent.Serialize(list)); + Assert.AreEqual("[42,'Sally',16.5]", S(ser)); + serpent.Indent=true; + ser = strip_header(serpent.Serialize(list)); + Assert.AreEqual("[\n 42,\n 'Sally',\n 16.5\n]", S(ser)); + } + + [Test] + public void TestSet() + { + // test with set literals + Serializer serpent = new Serializer(); + serpent.SetLiterals = true; + HashSet set = new HashSet(); + + // test empty set + byte[] ser = strip_header(serpent.Serialize(set)); + Assert.AreEqual("()", S(ser)); // empty set is serialized as a tuple. + serpent.Indent=true; + ser = strip_header(serpent.Serialize(set)); + Assert.AreEqual("()", S(ser)); // empty set is serialized as a tuple. + serpent.Indent=false; + + // test nonempty set + set.Add(42); + set.Add("Sally"); + set.Add(16.5); + ser = strip_header(serpent.Serialize(set)); + Assert.AreEqual("{42,'Sally',16.5}", S(ser)); + serpent.Indent=true; + ser = strip_header(serpent.Serialize(set)); + Assert.AreEqual("{\n 42,\n 'Sally',\n 16.5\n}", S(ser)); + + // test no set literals + serpent.Indent=false; + serpent.SetLiterals=false; + ser = strip_header(serpent.Serialize(set)); + Assert.AreEqual("(42,'Sally',16.5)", S(ser)); // needs to be tuple now + } + + [Test] + public void TestDictionary() + { + Serializer serpent = new Serializer(); + Parser p = new Parser(); + + // test empty dict + IDictionary ht = new Hashtable(); + byte[] ser = serpent.Serialize(ht); + Assert.AreEqual(B("{}"), strip_header(ser)); + string parsed = p.Parse(ser).Root.ToString(); + Assert.AreEqual("{}", parsed); + + // empty dict with indentation + serpent.Indent=true; + ser = serpent.Serialize(ht); + Assert.AreEqual(B("{}"), strip_header(ser)); + parsed = p.Parse(ser).Root.ToString(); + Assert.AreEqual("{}", parsed); + + // test dict with values + serpent.Indent=false; + ht = new Hashtable() { + {42, "fortytwo"}, + {"sixteen-and-half", 16.5}, + {"name", "Sally"}, + {"status", false} + }; + + ser = serpent.Serialize(ht); + Assert.AreEqual('}', ser[ser.Length-1]); + Assert.AreNotEqual(',', ser[ser.Length-2]); + parsed = p.Parse(ser).Root.ToString(); + Assert.AreEqual(69, parsed.Length); + + // test indentation + serpent.Indent=true; + ser = serpent.Serialize(ht); + Assert.AreEqual('}', ser[ser.Length-1]); + Assert.AreEqual('\n', ser[ser.Length-2]); + Assert.AreNotEqual(',', ser[ser.Length-3]); + string ser_str = S(strip_header(ser)); + Assert.IsTrue(ser_str.Contains("'name': 'Sally'")); + Assert.IsTrue(ser_str.Contains("'status': False")); + Assert.IsTrue(ser_str.Contains("42: 'fortytwo'")); + Assert.IsTrue(ser_str.Contains("'sixteen-and-half': 16.5")); + parsed = p.Parse(ser).Root.ToString(); + Assert.AreEqual(69, parsed.Length); + serpent.Indent=false; + + // generic Dictionary test + IDictionary mydict = new Dictionary { + { 1, "one" }, + { 2, "two" }, + }; + ser = serpent.Serialize(mydict); + ser_str = S(strip_header(ser)); + Assert.IsTrue(ser_str=="{2:'two',1:'one'}" || ser_str=="{1:'one',2:'two'}"); + } + + [Test] + public void TestBytes() + { + Serializer serpent = new Serializer(indent: true); + byte[] bytes = new byte[] { 97, 98, 99, 100, 101, 102 }; // abcdef + byte[] ser = serpent.Serialize(bytes); + Assert.AreEqual("{\n 'data': 'YWJjZGVm',\n 'encoding': 'base64'\n}", S(strip_header(ser))); + + Parser p = new Parser(); + string parsed = p.Parse(ser).Root.ToString(); + Assert.AreEqual(39, parsed.Length); + + var hashtable = new Hashtable { + {"data", "YWJjZGVm"}, + {"encoding", "base64"} + }; + byte[] bytes2 = Parser.ToBytes(hashtable); + Assert.AreEqual(bytes, bytes2); + + var dict = new Dictionary { + {"data", "YWJjZGVm"}, + {"encoding", "base64"} + }; + bytes2 = Parser.ToBytes(dict); + Assert.AreEqual(bytes, bytes2); + + var dict2 = new Dictionary { + {"data", "YWJjZGVm"}, + {"encoding", "base64"} + }; + bytes2 = Parser.ToBytes(dict2); + Assert.AreEqual(bytes, bytes2); + + dict["encoding"] = "base99"; + Assert.Throws(()=>Parser.ToBytes(dict)); + dict.Clear(); + Assert.Throws(()=>Parser.ToBytes(dict)); + dict.Clear(); + dict["data"] = "YWJjZGVm"; + Assert.Throws(()=>Parser.ToBytes(dict)); + dict.Clear(); + dict["encoding"] = "base64"; + Assert.Throws(()=>Parser.ToBytes(dict)); + Assert.Throws(()=>Parser.ToBytes(12345)); + Assert.Throws(()=>Parser.ToBytes(null)); + } + + [Test] + public void TestCollection() + { + ICollection intlist = new LinkedList(); + intlist.Add(42); + intlist.Add(43); + Serializer serpent = new Serializer(); + byte[] ser = serpent.Serialize(intlist); + ser = strip_header(ser); + Assert.AreEqual("[42,43]", S(ser)); + + ser=strip_header(serpent.Serialize(new int[] {42})); + Assert.AreEqual("(42,)", S(ser)); + ser=strip_header(serpent.Serialize(new int[] {42, 43})); + Assert.AreEqual("(42,43)", S(ser)); + + serpent.Indent=true; + ser = strip_header(serpent.Serialize(intlist)); + Assert.AreEqual("[\n 42,\n 43\n]", S(ser)); + ser=strip_header(serpent.Serialize(new int[] {42})); + Assert.AreEqual("(\n 42,\n)", S(ser)); + ser=strip_header(serpent.Serialize(new int[] {42, 43})); + Assert.AreEqual("(\n 42,\n 43\n)", S(ser)); + } + + + [Test] + public void TestIndentation() + { + var dict = new Dictionary(); + var list = new List() { + 1, + 2, + new string[] {"a", "b"} + }; + dict.Add("first", list); + dict.Add("second", new Dictionary { + {1, false} + }); + dict.Add("third", new HashSet { 3, 4} ); + + Serializer serpent = new Serializer(); + serpent.Indent=true; + byte[] ser = strip_header(serpent.Serialize(dict)); + string txt=@"{ + 'first': [ + 1, + 2, + ( + 'a', + 'b' + ) + ], + 'second': { + 1: False + }, + 'third': { + 3, + 4 + } +}"; + // bit of trickery to deal with Windows/Unix line ending differences + txt = txt.Replace("\n","\r\n"); + txt = txt.Replace("\r\r\n", "\r\n"); + string ser_txt = S(ser); + ser_txt = ser_txt.Replace("\n", "\r\n"); + ser_txt = ser_txt.Replace("\r\r\n", "\r\n"); + Assert.AreEqual(txt, ser_txt); + } + + [Test] + public void TestSorting() + { + Serializer serpent=new Serializer(); + object data = new List { 3, 2, 1}; + byte[] ser = strip_header(serpent.Serialize(data)); + Assert.AreEqual("[3,2,1]", S(ser)); + data = new int[] { 3,2,1 }; + ser = strip_header(serpent.Serialize(data)); + Assert.AreEqual("(3,2,1)", S(ser)); + + data = new HashSet { + 42, + "hi" + }; + serpent.Indent=true; + ser = strip_header(serpent.Serialize(data)); + Assert.IsTrue(S(ser)=="{\n 42,\n 'hi'\n}" || S(ser)=="{\n 'hi',\n 42\n}"); + + data = new Dictionary { + {5, "five"}, + {3, "three"}, + {1, "one"}, + {4, "four"}, + {2, "two"} + }; + serpent.Indent=true; + ser = strip_header(serpent.Serialize(data)); + Assert.AreEqual("{\n 1: 'one',\n 2: 'two',\n 3: 'three',\n 4: 'four',\n 5: 'five'\n}", S(ser)); + + data = new HashSet { + "x", + "y", + "z", + "c", + "b", + "a" + }; + serpent.Indent=true; + ser = strip_header(serpent.Serialize(data)); + Assert.AreEqual("{\n 'a',\n 'b',\n 'c',\n 'x',\n 'y',\n 'z'\n}", S(ser)); + } + + [Test] + public void TestClass() + { + Serializer.RegisterClass(typeof(SerializeTestClass), null); + Serializer serpent = new Serializer(indent: true); + + var obj = new SerializeTestClass() { + i = 99, + s = "hi", + x = 42 + }; + byte[] ser = strip_header(serpent.Serialize(obj)); + Assert.AreEqual("{\n '__class__': 'SerializeTestClass',\n 'i': 99,\n 'obj': None,\n 's': 'hi'\n}", S(ser)); + } + + [Test] + public void TestClass2() + { + Serializer.RegisterClass(typeof(SerializeTestClass), null); + Serializer serpent = new Serializer(indent: true, namespaceInClassName: true); + object obj = new SerializeTestClass() { + i = 99, + s = "hi", + x = 42 + }; + byte[] ser = strip_header(serpent.Serialize(obj)); + Assert.AreEqual("{\n '__class__': 'Razorvine.Serpent.Test.SerializeTestClass',\n 'i': 99,\n 'obj': None,\n 's': 'hi'\n}", S(ser)); + } + + protected IDictionary testclassConverter(object obj) + { + SerializeTestClass o = (SerializeTestClass) obj; + IDictionary result = new Hashtable(); + result["__class@__"] = o.GetType().Name+"@"; + result["i@"] = o.i; + result["s@"] = o.s; + result["x@"] = o.x; + return result; + } + + [Test] + public void TestCustomClassDict() + { + Serializer.RegisterClass(typeof(SerializeTestClass), testclassConverter); + Serializer serpent = new Serializer(indent: true); + + var obj = new SerializeTestClass() { + i = 99, + s = "hi", + x = 42 + }; + byte[] ser = strip_header(serpent.Serialize(obj)); + Assert.AreEqual("{\n '__class@__': 'SerializeTestClass@',\n 'i@': 99,\n 's@': 'hi',\n 'x@': 42\n}", S(ser)); + } + + [Test] + public void TestStruct() + { + Serializer serpent = new Serializer(indent: true); + + var obj2 = new SerializeTestStruct() { + i = 99, + s = "hi", + x = 42 + }; + byte[] ser = strip_header(serpent.Serialize(obj2)); + Assert.AreEqual("{\n '__class__': 'SerializeTestStruct',\n 'i': 99,\n 's': 'hi'\n}", S(ser)); + } + + [Test] + public void TestStruct2() + { + Serializer serpent = new Serializer(indent: true, namespaceInClassName: true); + + var obj2 = new SerializeTestStruct() { + i = 99, + s = "hi", + x = 42 + }; + byte[] ser = strip_header(serpent.Serialize(obj2)); + Assert.AreEqual("{\n '__class__': 'Razorvine.Serpent.Test.SerializeTestStruct',\n 'i': 99,\n 's': 'hi'\n}", S(ser)); + } + + [Test] + public void TestAnonymousClass() + { + Serializer serpent = new Serializer(indent: true); + Object obj = new { + Name="Harry", + Age=33, + Country="NL" + }; + + byte[] ser = strip_header(serpent.Serialize(obj)); + Assert.AreEqual("{\n 'Age': 33,\n 'Country': 'NL',\n 'Name': 'Harry'\n}", S(ser)); + } + + [Test] + public void TestDateTime() + { + Serializer serpent = new Serializer(); + + DateTime date = new DateTime(2013, 1, 20, 23, 59, 45, 999, DateTimeKind.Local); + byte[] ser = strip_header(serpent.Serialize(date)); + Assert.AreEqual("'2013-01-20T23:59:45.999'", S(ser)); + + date = new DateTime(2013, 1, 20, 23, 59, 45, 999, DateTimeKind.Utc); + ser = strip_header(serpent.Serialize(date)); + Assert.AreEqual("'2013-01-20T23:59:45.999'", S(ser)); + + date = new DateTime(2013, 1, 20, 23, 59, 45, 999, DateTimeKind.Unspecified); + ser = strip_header(serpent.Serialize(date)); + Assert.AreEqual("'2013-01-20T23:59:45.999'", S(ser)); + + date = new DateTime(2013, 1, 20, 23, 59, 45); + ser = strip_header(serpent.Serialize(date)); + Assert.AreEqual("'2013-01-20T23:59:45'", S(ser)); + + TimeSpan timespan = new TimeSpan(1, 10, 20, 30, 999); + ser = strip_header(serpent.Serialize(timespan)); + Assert.AreEqual("123630.999", S(ser)); + } + + [Test] + public void TestDateTimeOffset() + { + Serializer serpent = new Serializer(); + + DateTimeOffset date = new DateTimeOffset(2013, 1, 20, 23, 59, 45, 999, TimeSpan.FromHours(+2)); + byte[] ser = strip_header(serpent.Serialize(date)); + Assert.AreEqual("'2013-01-20T23:59:45.999+02:00'", S(ser)); + + date = new DateTimeOffset(2013, 5, 10, 13, 59, 45, TimeSpan.FromHours(+2)); + ser = strip_header(serpent.Serialize(date)); + Assert.AreEqual("'2013-05-10T13:59:45+02:00'", S(ser)); + } + + [Test] + public void TestException() + { + Exception x = new ApplicationException("errormessage"); + Serializer serpent = new Serializer(indent:true); + byte[] ser = strip_header(serpent.Serialize(x)); + Assert.AreEqual("{\n '__class__': 'ApplicationException',\n '__exception__': True,\n 'args': (\n 'errormessage',\n ),\n 'attributes': {}\n}", S(ser)); + + x.Data["custom_attribute"]=999; + ser = strip_header(serpent.Serialize(x)); + Assert.AreEqual("{\n '__class__': 'ApplicationException',\n '__exception__': True,\n 'args': (\n 'errormessage',\n ),\n 'attributes': {\n 'custom_attribute': 999\n }\n}", S(ser)); + } + + [Test] + public void TestExceptionWithNamespace() + { + Exception x = new ApplicationException("errormessage"); + Serializer serpent = new Serializer(indent:true, namespaceInClassName: true); + byte[] ser = strip_header(serpent.Serialize(x)); + Assert.AreEqual("{\n '__class__': 'System.ApplicationException',\n '__exception__': True,\n 'args': (\n 'errormessage',\n ),\n 'attributes': {}\n}", S(ser)); + } + + enum FooType { + Foobar, + Jarjar + } + + [Test] + public void TestEnum() + { + FooType e = FooType.Jarjar; + Serializer serpent = new Serializer(); + byte[] ser = strip_header(serpent.Serialize(e)); + Assert.AreEqual("'Jarjar'", S(ser)); + } + + + interface IBaseInterface {}; + interface ISubInterface : IBaseInterface {}; + class BaseClassWithInterface : IBaseInterface {}; + class SubClassWithInterface : BaseClassWithInterface, ISubInterface {}; + class BaseClass {}; + class SubClass : BaseClass {}; + abstract class AbstractBaseClass {}; + class ConcreteSubClass : AbstractBaseClass {}; + + protected IDictionary AnyClassSerializer(object arg) + { + IDictionary result = new Hashtable(); + result["(SUB)CLASS"] = arg.GetType().Name; + return result; + } + + [Test] + public void testAbstractBaseClassHierarchyPickler() + { + ConcreteSubClass c = new ConcreteSubClass(); + Serializer serpent = new Serializer(); + serpent.Serialize(c); + + Serializer.RegisterClass(typeof(AbstractBaseClass), AnyClassSerializer); + byte[] data = serpent.Serialize(c); + Assert.AreEqual("{'(SUB)CLASS':'ConcreteSubClass'}", S(strip_header(data))); + } + + [Test] + public void testInterfaceHierarchyPickler() + { + BaseClassWithInterface b = new BaseClassWithInterface(); + SubClassWithInterface sub = new SubClassWithInterface(); + Serializer serpent = new Serializer(); + serpent.Serialize(b); + serpent.Serialize(sub); + Serializer.RegisterClass(typeof(IBaseInterface), AnyClassSerializer); + byte[] data = serpent.Serialize(b); + Assert.AreEqual("{'(SUB)CLASS':'BaseClassWithInterface'}", S(strip_header(data))); + data = serpent.Serialize(sub); + Assert.AreEqual("{'(SUB)CLASS':'SubClassWithInterface'}", S(strip_header(data))); + } + } + + public class SerializeTestClass + { + public int x; + public string s {get; set;} + public int i {get; set;} + public object obj {get; set;} + + } + + public struct SerializeTestStruct + { + public int x; + public string s {get; set;} + public int i {get; set;} + } +} diff --git a/java/src/main/java/net/razorvine/serpent/LibraryVersion.java b/java/src/main/java/net/razorvine/serpent/LibraryVersion.java index b7a0159..71e5de8 100644 --- a/java/src/main/java/net/razorvine/serpent/LibraryVersion.java +++ b/java/src/main/java/net/razorvine/serpent/LibraryVersion.java @@ -1,12 +1,12 @@ -/** - * Serpent, a Python literal expression serializer/deserializer - * (a.k.a. Python's ast.literal_eval in Java) - * Software license: "MIT software license". See http://opensource.org/licenses/MIT - * @author Irmen de Jong (irmen@razorvine.net) - */ - -package net.razorvine.serpent; - -public final class LibraryVersion { - public static final String VERSION = "1.15"; -} +/** + * Serpent, a Python literal expression serializer/deserializer + * (a.k.a. Python's ast.literal_eval in Java) + * Software license: "MIT software license". See http://opensource.org/licenses/MIT + * @author Irmen de Jong (irmen@razorvine.net) + */ + +package net.razorvine.serpent; + +public final class LibraryVersion { + public static final String VERSION = "1.16"; +} diff --git a/java/src/main/java/net/razorvine/serpent/package-info.java b/java/src/main/java/net/razorvine/serpent/package-info.java index a473a31..78e82bb 100644 --- a/java/src/main/java/net/razorvine/serpent/package-info.java +++ b/java/src/main/java/net/razorvine/serpent/package-info.java @@ -1,10 +1,10 @@ -/** - * Serpent, a Python literal expression serializer/deserializer - * (a.k.a. Python's ast.literal_eval in Java) - * Software license: "MIT software license". See http://opensource.org/licenses/MIT - * @author Irmen de Jong (irmen@razorvine.net) - * @version 1.15 - */ - -package net.razorvine.serpent; - +/** + * Serpent, a Python literal expression serializer/deserializer + * (a.k.a. Python's ast.literal_eval in Java) + * Software license: "MIT software license". See http://opensource.org/licenses/MIT + * @author Irmen de Jong (irmen@razorvine.net) + * @version 1.16 + */ + +package net.razorvine.serpent; + diff --git a/java/src/test/java/net/razorvine/serpent/test/ParserTest.java b/java/src/test/java/net/razorvine/serpent/test/ParserTest.java index a2cfaa6..b63482d 100644 --- a/java/src/test/java/net/razorvine/serpent/test/ParserTest.java +++ b/java/src/test/java/net/razorvine/serpent/test/ParserTest.java @@ -1,847 +1,895 @@ -/** - * Serpent, a Python literal expression serializer/deserializer - * (a.k.a. Python's ast.literal_eval in Java) - * Software license: "MIT software license". See http://opensource.org/licenses/MIT - * @author Irmen de Jong (irmen@razorvine.net) - */ - -package net.razorvine.serpent.test; - -import static org.junit.Assert.*; - -import java.io.DataInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.math.BigInteger; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; - -import net.razorvine.serpent.ObjectifyVisitor; -import net.razorvine.serpent.ParseException; -import net.razorvine.serpent.Parser; -import net.razorvine.serpent.SeekableStringReader; -import net.razorvine.serpent.ast.*; - -import org.junit.Ignore; -import org.junit.Test; - - -public class ParserTest -{ - @Test - public void TestBasic() - { - Parser p = new Parser(); - assertNull(p.parse((String)null).root); - assertNull(p.parse("").root); - assertNotNull(p.parse("# comment\n42\n").root); - } - - @Test - public void TestComments() - { - Parser p = new Parser(); - - Ast ast = p.parse("[ 1, 2 ]"); // no header whatsoever - ObjectifyVisitor visitor = new ObjectifyVisitor(); - ast.accept(visitor); - Object obj = visitor.getObject(); - - List expected = new ArrayList(); - expected.add(1); - expected.add(2); - assertEquals(expected, obj); - expected = new LinkedList(); - expected.add(1); - expected.add(2); - assertEquals(expected, obj); - - ast = p.parse("# serpent utf-8 python2.7\n"+ -"[ 1, 2,\n"+ -" # some comments here\n"+ -" 3, 4] # more here\n"+ -"# and here.\n"); - visitor = new ObjectifyVisitor(); - ast.accept(visitor); - obj = visitor.getObject(); - expected = new LinkedList(); - expected.add(1); - expected.add(2); - expected.add(3); - expected.add(4); - assertEquals(expected, obj); - } - - @Test - public void TestPrimitives() - { - Parser p = new Parser(); - assertEquals(new IntegerNode(42), p.parse("42").root); - assertEquals(new IntegerNode(-42), p.parse("-42").root); - assertEquals(new DoubleNode(42.331), p.parse("42.331").root); - assertEquals(new DoubleNode(-42.331), p.parse("-42.331").root); - assertEquals(new DoubleNode(-1.2e19), p.parse("-1.2e+19").root); - assertEquals(new DoubleNode(0.0004), p.parse("4e-4").root); - assertEquals(new DoubleNode(40000), p.parse("4e4").root); - assertEquals(new BooleanNode(true), p.parse("True").root); - assertEquals(new BooleanNode(false), p.parse("False").root); - assertEquals(NoneNode.Instance, p.parse("None").root); - - // long ints - assertEquals(new BigIntNode(new BigInteger("123456789123456789123456789")), p.parse("123456789123456789123456789").root); - assertFalse(new LongNode(52).equals(p.parse("52").root)); - assertEquals(new LongNode(123456789123456789L), p.parse("123456789123456789").root); - - assertEquals(BigIntNode.class, p.parse("12345678912345678912345678912345678978571892375798273578927389758378467693485903859038593453475897349587348957893457983475983475893475893785732957398475").root.getClass()); - } - - @Test - public void TestWeirdFloats() - { - Parser p = new Parser(); - TupleNode tuple = (TupleNode) p.parse("(1e30000,-1e30000,(1e30000+3.4j),{'__class__':'float','value':'nan'})").root; - assertEquals(4, tuple.elements.size()); - DoubleNode d = (DoubleNode) tuple.elements.get(0); - assertTrue(Double.isInfinite(d.value)); - d = (DoubleNode) tuple.elements.get(1); - assertTrue(Double.isInfinite(d.value)); - assertTrue(d.value < 0.0); - ComplexNumberNode c = (ComplexNumberNode) tuple.elements.get(2); - assertTrue(Double.isInfinite(c.real)); - assertEquals(3.4, c.imaginary, 0); - d = (DoubleNode) tuple.elements.get(3); - assertTrue(Double.isNaN(d.value)); - } - - @Test - public void TestEquality() - { - INode n1, n2; - - n1 = new IntegerNode(42); - n2 = new IntegerNode(42); - assertEquals(n1, n2); - n2 = new IntegerNode(43); - assertFalse(n1.equals(n2)); - - n1 = new StringNode("foo"); - n2 = new StringNode("foo"); - assertEquals(n1, n2); - n2 = new StringNode("bar"); - assertFalse(n1.equals(n2)); - - ComplexNumberNode cn1 = new ComplexNumberNode(); - cn1.real=1.1; cn1.imaginary=2.2; - ComplexNumberNode cn2 = new ComplexNumberNode(); - cn2.real=1.1; cn2.imaginary=2.2; - assertEquals(cn1, cn2); - cn2 = new ComplexNumberNode(); - cn2.real=1.1; cn2.imaginary=3.3; - assertFalse(cn1.equals(cn2)); - - KeyValueNode kvn1=new KeyValueNode(); - kvn1.key = new IntegerNode(42); - kvn1.value = new IntegerNode(42); - KeyValueNode kvn2=new KeyValueNode(); - kvn2.key=new IntegerNode(42); - kvn2.value=new IntegerNode(42); - assertEquals(kvn1, kvn2); - kvn1=new KeyValueNode(); - kvn1.key=new IntegerNode(43); - kvn1.value=new IntegerNode(43); - assertFalse(kvn1.equals(kvn2)); - - n1=NoneNode.Instance; - n2=NoneNode.Instance; - assertEquals(n1, n2); - n2=new IntegerNode(42); - assertFalse(n1.equals(n2)); - - DictNode dn1 = new DictNode(); - kvn1 = new KeyValueNode(); - kvn1.key = new IntegerNode(42); - kvn1.value = new IntegerNode(42); - dn1.elements.add(kvn1); - DictNode dn2=new DictNode(); - kvn1 = new KeyValueNode(); - kvn1.key =new IntegerNode(42); - kvn1.value = new IntegerNode(42); - dn2.elements.add(kvn1); - assertEquals(dn1, dn2); - - dn2=new DictNode(); - kvn1 = new KeyValueNode(); - kvn1.key = new IntegerNode(42); - kvn1.value = new IntegerNode(43); - dn2.elements.add(kvn1); - assertFalse(dn1.equals(dn2)); - - ListNode ln1=new ListNode(); - ln1.elements.add(new IntegerNode(42)); - ListNode ln2=new ListNode(); - ln2.elements.add(new IntegerNode(42)); - assertEquals(ln1,ln2); - ln2=new ListNode(); - ln2.elements.add(new IntegerNode(43)); - assertFalse(ln1.equals(ln2)); - - SetNode sn1=new SetNode(); - sn1.elements.add(new IntegerNode(42)); - SetNode sn2=new SetNode(); - sn2.elements.add(new IntegerNode(42)); - assertEquals(sn1,sn2); - sn2=new SetNode(); - sn2.elements.add(new IntegerNode(43)); - assertFalse(sn1.equals(sn2)); - - TupleNode tn1=new TupleNode(); - tn1.elements.add(new IntegerNode(42)); - TupleNode tn2=new TupleNode(); - tn2.elements.add(new IntegerNode(42)); - assertEquals(tn1,tn2); - tn2=new TupleNode(); - tn2.elements.add(new IntegerNode(43)); - assertFalse(tn1.equals(tn2)); - - } - - @Test - public void TestPrintSingle() - { - Parser p = new Parser(); - - // primitives - assertEquals("42", p.parse("42").root.toString()); - assertEquals("-42.331", p.parse("-42.331").root.toString()); - assertEquals("-42.0", p.parse("-42.0").root.toString()); - assertEquals("-2.0E20", p.parse("-2E20").root.toString()); - assertEquals("2.0", p.parse("2.0").root.toString()); - assertEquals("1.2E19", p.parse("1.2e19").root.toString()); - assertEquals("True", p.parse("True").root.toString()); - assertEquals("'hello'", p.parse("'hello'").root.toString()); - assertEquals("'\\n'", p.parse("'\n'").root.toString()); - assertEquals("'\\''", p.parse("'\\''").root.toString()); - assertEquals("'\"'", p.parse("'\\\"'").root.toString()); - assertEquals("'\"'", p.parse("'\"'").root.toString()); - assertEquals("'\\\\'", p.parse("'\\\\'").root.toString()); - assertEquals("None", p.parse("None").root.toString()); - String ustr = "'\u20ac\u2603'"; - assertEquals(ustr, p.parse(ustr).root.toString()); - - // complex - assertEquals("(0.0+2.0j)", p.parse("2j").root.toString()); - assertEquals("(-1.1-2.2j)", p.parse("(-1.1-2.2j)").root.toString()); - assertEquals("(1.1+2.2j)", p.parse("(1.1+2.2j)").root.toString()); - - // long int - assertEquals("123456789123456789123456789", p.parse("123456789123456789123456789").root.toString()); - } - - @Test - public void TestPrintSeq() - { - Parser p=new Parser(); - - //tuple - assertEquals("()", p.parse("()").root.toString()); - assertEquals("(42,)", p.parse("(42,)").root.toString()); - assertEquals("(42,43)", p.parse("(42,43)").root.toString()); - - // list - assertEquals("[]", p.parse("[]").root.toString()); - assertEquals("[42]", p.parse("[42]").root.toString()); - assertEquals("[42,43]", p.parse("[42,43]").root.toString()); - - // set - assertEquals("{42}", p.parse("{42}").root.toString()); - assertEquals("{42,43}", p.parse("{42,43,43,43}").root.toString()); - - // dict - assertEquals("{}", p.parse("{}").root.toString()); - assertEquals("{'a':42}", p.parse("{'a': 42}").root.toString()); - - String result = p.parse("{'a': 42, 'b': 43}").root.toString(); - assertTrue(result.equals("{'a':42,'b':43}") || result.equals("{'b':43,'a':42}")); - result = p.parse("{'a': 42, 'b': 43, 'b': 44, 'b': 45}").root.toString(); - assertTrue(result.equals("{'a':42,'b':45}") || result.equals("{'b':45,'a':42}")); - } - - @Test(expected=ParseException.class) - public void TestInvalidPrimitive1() - { - new Parser().parse("1+2"); - } - - @Test(expected=ParseException.class) - public void TestInvalidPrimitive2() - { - new Parser().parse("1-2"); - } - - @Test(expected=ParseException.class) - public void TestInvalidPrimitive3() - { - new Parser().parse("1.1+2.2"); - } - - @Test(expected=ParseException.class) - public void TestInvalidPrimitive4() - { - new Parser().parse("1.1-2.2"); - } - - @Test(expected=ParseException.class) - public void TestInvalidPrimitive5() - { - new Parser().parse("True+2"); - } - - @Test(expected=ParseException.class) - public void TestInvalidPrimitive6() - { - new Parser().parse("False-2"); - } - - @Test(expected=ParseException.class) - public void TestInvalidPrimitive7() - { - new Parser().parse("3j+2"); - } - - @Test(expected=ParseException.class) - public void TestInvalidPrimitive8() - { - new Parser().parse("3j-2"); - } - - @Test(expected=ParseException.class) - public void TestInvalidPrimitive9() - { - new Parser().parse("None+2"); - } - - @Test(expected=ParseException.class) - public void TestInvalidPrimitive10() - { - new Parser().parse("None-2"); - } - - @Test(expected=ParseException.class) - public void TestInvalidTuple1() - { - new Parser().parse("(42,43]"); - } - - @Test(expected=ParseException.class) - public void TestInvalidTuple2() - { - new Parser().parse("()@"); - } - - @Test(expected=ParseException.class) - public void TestInvalidTuple3() - { - new Parser().parse("(42,43)@"); - } - - @Test(expected=ParseException.class) - public void TestInvalidList2() - { - new Parser().parse("[42,43}"); - } - - @Test(expected=ParseException.class) - public void TestInvalidList3() - { - new Parser().parse("[]@"); - } - - @Test(expected=ParseException.class) - public void TestInvalidList4() - { - new Parser().parse("[42,43]@"); - } - - @Test(expected=ParseException.class) - public void TestInvalidSet2() - { - new Parser().parse("{42,43]"); - } - - @Test(expected=ParseException.class) - public void TestInvalidSet3() - { - new Parser().parse("{42,43}@"); - } - - @Test(expected=ParseException.class) - public void TestInvalidDict2() - { - new Parser().parse("{'key': 42]"); - } - - @Test(expected=ParseException.class) - public void TestInvalidDict3() - { - new Parser().parse("{}@"); - } - - @Test(expected=ParseException.class) - public void TestInvalidDict4() - { - new Parser().parse("{'key': 42}@"); - } - - @Test - public void TestComplex() - { - Parser p = new Parser(); - ComplexNumberNode cplx = new ComplexNumberNode(); - cplx.real = 4.2; - cplx.imaginary = 3.2; - ComplexNumberNode cplx2 = new ComplexNumberNode(); - cplx2.real = 4.2; - cplx2.imaginary = 99; - assertFalse(cplx.equals(cplx2)); - cplx2.imaginary = 3.2; - assertEquals(cplx, cplx2); - - assertEquals(cplx, p.parse("(4.2+3.2j)").root); - cplx.real = 0; - assertEquals(cplx, p.parse("(0+3.2j)").root); - assertEquals(cplx, p.parse("3.2j").root); - assertEquals(cplx, p.parse("+3.2j").root); - cplx.imaginary = -3.2; - assertEquals(cplx, p.parse("-3.2j").root); - cplx.real = -9.9; - assertEquals(cplx, p.parse("(-9.9-3.2j)").root); - - cplx.real = 2; - cplx.imaginary = 3; - assertEquals(cplx, p.parse("(2+3j)").root); - cplx.imaginary = -3; - assertEquals(cplx, p.parse("(2-3j)").root); - cplx.real = 0; - assertEquals(cplx, p.parse("-3j").root); - } - - @Test - public void TestPrimitivesStuffAtEnd() - { - Parser p = new Parser(); - assertEquals(new IntegerNode(42), p.parseSingle(new SeekableStringReader("42@"))); - assertEquals(new DoubleNode(42.331), p.parseSingle(new SeekableStringReader("42.331@"))); - assertEquals(new BooleanNode(true), p.parseSingle(new SeekableStringReader("True@"))); - assertEquals(NoneNode.Instance, p.parseSingle(new SeekableStringReader("None@"))); - ComplexNumberNode cplx = new ComplexNumberNode(); - cplx.real=4; - cplx.imaginary=3; - assertEquals(cplx, p.parseSingle(new SeekableStringReader("(4+3j)@"))); - cplx.real=0; - assertEquals(cplx, p.parseSingle(new SeekableStringReader("3j@"))); - } - - @Test - public void TestStrings() - { - Parser p = new Parser(); - assertEquals(new StringNode("hello"), p.parse("'hello'").root); - assertEquals(new StringNode("hello"), p.parse("\"hello\"").root); - assertEquals(new StringNode("\\"), p.parse("'\\\\'").root); - assertEquals(new StringNode("\\"), p.parse("\"\\\\\"").root); - assertEquals(new StringNode("'"), p.parse("\"'\"").root); - assertEquals(new StringNode("\""), p.parse("'\"'").root); - assertEquals(new StringNode("tab\tnewline\n."), p.parse("'tab\\tnewline\\n.'").root); - } - - @Test - public void TestUnicode() throws UnsupportedEncodingException - { - Parser p = new Parser(); - String str = "'\u20ac\u2603'"; - assertEquals(0x20ac, str.charAt(1)); - assertEquals(0x2603, str.charAt(2)); - byte[] bytes = str.getBytes("utf-8"); - - String value = "\u20ac\u2603"; - assertEquals(new StringNode(value), p.parse(str).root); - assertEquals(new StringNode(value), p.parse(bytes).root); - } - - @Test - public void TestWhitespace() - { - Parser p = new Parser(); - assertEquals(new IntegerNode(42), p.parse(" 42 ").root); - assertEquals(new IntegerNode(42), p.parse(" 42 ").root); - assertEquals(new IntegerNode(42), p.parse("\t42\r\n").root); - assertEquals(new IntegerNode(42), p.parse(" \t 42 \r \n ").root); - assertEquals(new StringNode(" string value "), p.parse(" ' string value ' ").root); - try { - p.parse(" ( 42 , 43 , "); // missing tuple close ) - fail("expected parse error"); - } catch (ParseException x) { - //ok - } - - try { - p.parse(" [ 42 , 43 , "); // missing list close ) - fail("expected parse error"); - } catch (ParseException x) { - //ok - } - - try { - p.parse(" { 42 , 43 , "); // missing set close ) - fail("expected parse error"); - } catch (ParseException x) { - //ok - } - - try { - p.parse(" { 'a' : 4 2 , "); // missing dict close ) - fail("expected parse error"); - } catch (ParseException x) { - //ok - } - - Ast ast = p.parse(" ( 42 , ( 'x', 'y' ) ) "); - TupleNode tuple = (TupleNode) ast.root; - assertEquals(new IntegerNode(42), tuple.elements.get(0)); - tuple = (TupleNode) tuple.elements.get(1); - assertEquals(new StringNode("x"), tuple.elements.get(0)); - assertEquals(new StringNode("y"), tuple.elements.get(1)); - - p.parse(" ( 52 , ) "); - p.parse(" [ 52 ] "); - p.parse(" { 'a' : 42 } "); - p.parse(" { 52 } "); - } - - @Test - public void TestTuple() - { - Parser p = new Parser(); - TupleNode tuple = new TupleNode(); - TupleNode tuple2 = new TupleNode(); - assertEquals(tuple, tuple2); - - tuple.elements.add(new IntegerNode(42)); - tuple2.elements.add(new IntegerNode(99)); - assertFalse(tuple.equals(tuple2)); - tuple2.elements.clear(); - tuple2.elements.add(new IntegerNode(42)); - assertEquals(tuple, tuple2); - tuple2.elements.add(new IntegerNode(43)); - tuple2.elements.add(new IntegerNode(44)); - assertFalse(tuple.equals(tuple2)); - - assertEquals(new TupleNode(), p.parse("()").root); - assertEquals(tuple, p.parse("(42,)").root); - assertEquals(tuple2, p.parse("( 42,43, 44 )").root); - } - - @Test - public void TestList() - { - Parser p = new Parser(); - ListNode list = new ListNode(); - ListNode list2 = new ListNode(); - assertEquals(list, list2); - - list.elements.add(new IntegerNode(42)); - list2.elements.add(new IntegerNode(99)); - assertFalse(list.equals(list2)); - list2.elements.clear(); - list2.elements.add(new IntegerNode(42)); - assertEquals(list, list2); - list2.elements.add(new IntegerNode(43)); - list2.elements.add(new IntegerNode(44)); - assertFalse(list.equals(list2)); - - assertEquals(new ListNode(), p.parse("[]").root); - assertEquals(list, p.parse("[42]").root); - assertEquals(list2, p.parse("[ 42,43, 44 ]").root); - - } - - @Test - public void TestSet() - { - Parser p = new Parser(); - SetNode set1 = new SetNode(); - SetNode set2 = new SetNode(); - assertEquals(set1, set2); - - set1.elements.add(new IntegerNode(42)); - set2.elements.add(new IntegerNode(99)); - assertFalse(set1.equals(set2)); - set2.elements.clear(); - set2.elements.add(new IntegerNode(42)); - assertEquals(set1, set2); - - set2.elements.add(new IntegerNode(43)); - set2.elements.add(new IntegerNode(44)); - assertFalse(set1.equals(set2)); - - assertEquals(set1, p.parse("{42}").root); - assertEquals(set2, p.parse("{ 42,43, 44 }").root); - - set1 = (SetNode) p.parse("{'first','second','third','fourth','fifth','second', 'first', 'third', 'third' }").root; - assertTrue(set1.elements.contains(new StringNode("first"))); - assertTrue(set1.elements.contains(new StringNode("second"))); - assertTrue(set1.elements.contains(new StringNode("third"))); - assertTrue(set1.elements.contains(new StringNode("fourth"))); - assertTrue(set1.elements.contains(new StringNode("fifth"))); - assertEquals(5, set1.elements.size()); - } - - @Test - public void TestDict() - { - Parser p = new Parser(); - DictNode dict1 = new DictNode(); - DictNode dict2 = new DictNode(); - assertEquals(dict1, dict2); - - KeyValueNode kv1 = new KeyValueNode(); - kv1.key = new StringNode("key"); - kv1.value = new IntegerNode(42); - KeyValueNode kv2 = new KeyValueNode(); - kv2.key=new StringNode("key"); - kv2.value=new IntegerNode(99); - assertFalse(kv1.equals(kv2)); - kv2.value = new IntegerNode(42); - assertEquals(kv1, kv2); - - kv1=new KeyValueNode(); - kv1.key=new StringNode("key1"); - kv1.value=new IntegerNode(42); - dict1.elements.add(kv1); - kv1=new KeyValueNode(); - kv1.key=new StringNode("key1"); - kv1.value=new IntegerNode(99); - dict2.elements.add(kv1); - assertFalse(dict1.equals(dict2)); - - dict2.elements.clear(); - - kv1=new KeyValueNode(); - kv1.key=new StringNode("key1"); - kv1.value=new IntegerNode(42); - dict2.elements.add(kv1); - assertEquals(dict1, dict2); - - kv1=new KeyValueNode(); - kv1.key=new StringNode("key2"); - kv1.value=new IntegerNode(43); - dict2.elements.add(kv1); - kv1=new KeyValueNode(); - kv1.key=new StringNode("key3"); - kv1.value=new IntegerNode(44); - dict2.elements.add(kv1); - assertFalse(dict1.equals(dict2)); - - assertEquals(new DictNode(), p.parse("{}").root); - assertEquals(dict1, p.parse("{'key1': 42}").root); - assertEquals(dict2, p.parse("{'key1': 42, 'key2': 43, 'key3':44}").root); - - dict1 = (DictNode) p.parse("{'a': 1, 'b': 2, 'c': 3, 'c': 4, 'c': 5, 'c': 6}").root; - - kv1 = new KeyValueNode(); - kv1.key=new StringNode("c"); - kv1.value=new IntegerNode(6); - assertTrue(dict1.elements.contains(kv1)); - assertEquals(3, dict1.elements.size()); - } - - @Test - public void TestKeyValueEquality() - { - KeyValueNode kv1=new KeyValueNode(); - kv1.key=new StringNode("key1"); - kv1.value=new IntegerNode(42); - - KeyValueNode kv2=new KeyValueNode(); - kv2.key=new StringNode("key1"); - kv2.value=new IntegerNode(42); - - assertEquals(kv1, kv2); - kv2.value=new IntegerNode(43); - assertFalse(kv1.equals(kv2)); - } - - @Test - public void TestDictEquality() - { - DictNode dict1 = new DictNode(); - KeyValueNode kv=new KeyValueNode(); - kv.key=new StringNode("key1"); - kv.value=new IntegerNode(42); - dict1.elements.add(kv); - - DictNode dict2 = new DictNode(); - kv=new KeyValueNode(); - kv.key=new StringNode("key1"); - kv.value=new IntegerNode(42); - dict2.elements.add(kv); - - assertEquals(dict1, dict2); - - - kv=new KeyValueNode(); - kv.key=new StringNode("key2"); - kv.value=new IntegerNode(43); - dict1.elements.add(kv); - kv=new KeyValueNode(); - kv.key=new StringNode("key3"); - kv.value=new IntegerNode(44); - dict1.elements.add(kv); - - dict2 = new DictNode(); - kv=new KeyValueNode(); - kv.key=new StringNode("key2"); - kv.value=new IntegerNode(43); - dict2.elements.add(kv); - kv=new KeyValueNode(); - kv.key=new StringNode("key3"); - kv.value=new IntegerNode(44); - dict2.elements.add(kv); - kv=new KeyValueNode(); - kv.key=new StringNode("key1"); - kv.value=new IntegerNode(42); - dict2.elements.add(kv); - - assertTrue(dict1.equals(dict2)); - assertEquals(dict1, dict2); - kv=new KeyValueNode(); - kv.key=new StringNode("key4"); - kv.value=new IntegerNode(45); - dict2.elements.add(kv); - assertFalse(dict1.equals(dict2)); - } - - @Test - public void TestSetEquality() - { - SetNode set1 = new SetNode(); - set1.elements.add(new IntegerNode(1)); - set1.elements.add(new IntegerNode(2)); - set1.elements.add(new IntegerNode(3)); - - SetNode set2 = new SetNode(); - set2.elements.add(new IntegerNode(2)); - set2.elements.add(new IntegerNode(3)); - set2.elements.add(new IntegerNode(1)); - - assertEquals(set1, set2); - set2.elements.add(new IntegerNode(0)); - assertFalse(set1.equals(set2)); - } - - @Test - public void TestFile() throws IOException - { - Parser p = new Parser(); - File testdatafile = new File("src/test/java/testserpent.utf8.bin"); - byte[] ser = new byte[(int) testdatafile.length()]; - FileInputStream fis=new FileInputStream(testdatafile); - DataInputStream dis = new DataInputStream(fis); - dis.readFully(ser); - dis.close(); - fis.close(); - - Ast ast = p.parse(ser); - - String expr = ast.toString(); - Ast ast2 = p.parse(expr); - String expr2 = ast2.toString(); - assertEquals(expr.length(), expr2.length()); - - StringBuilder sb= new StringBuilder(); - Walk(ast.root, sb); - String walk1 = sb.toString(); - sb= new StringBuilder(); - Walk(ast2.root, sb); - String walk2 = sb.toString(); - assertEquals(walk1.length(), walk2.length()); - - // TODO assertEquals(ast.root, ast2.root); - ast = p.parse(expr2); - // TODO assertEquals(ast.root, ast2.root); - } - - @Test - public void TestTrailingCommas() throws IOException - { - Parser p = new Parser(); - INode result; - result = p.parse("[1,2,3, ]").root; - result = p.parse("[1,2,3 , ]").root; - result = p.parse("[1,2,3,]").root; - assertEquals("[1,2,3]", result.toString()); - result = p.parse("(1,2,3, )").root; - result = p.parse("(1,2,3 , )").root; - result = p.parse("(1,2,3,)").root; - assertEquals("(1,2,3)", result.toString()); - - // for dict and set the asserts are a bit more complex - // we cannot simply convert to string because the order of elts is undefined. - - result = p.parse("{'a':1, 'b':2, 'c':3, }").root; - result = p.parse("{'a':1, 'b':2, 'c':3 , }").root; - result = p.parse("{'a':1, 'b':2, 'c':3,}").root; - DictNode dict = (DictNode) result; - assertEquals(3, dict.elements.size()); - Set elts = dict.elementsAsSet(); - assertTrue(elts.contains(new KeyValueNode(new StringNode("a"), new IntegerNode(1)))); - assertTrue(elts.contains(new KeyValueNode(new StringNode("b"), new IntegerNode(2)))); - assertTrue(elts.contains(new KeyValueNode(new StringNode("c"), new IntegerNode(3)))); - result = p.parse("{1,2,3, }").root; - result = p.parse("{1,2,3 , }").root; - result = p.parse("{1,2,3,}").root; - SetNode set = (SetNode) result; - assertEquals(3, set.elements.size()); - elts = set.elementsAsSet(); - assertTrue(elts.contains(new IntegerNode(1))); - assertTrue(elts.contains(new IntegerNode(2))); - assertTrue(elts.contains(new IntegerNode(3))); - } - - - @Test - @Ignore("can't get the ast compare to succeed :(") - public void TestAstEquals() throws IOException - { - Parser p = new Parser(); - File testdatafile = new File("test/testserpent.utf8.bin"); - byte[] ser = new byte[(int) testdatafile.length()]; - FileInputStream fis=new FileInputStream(testdatafile); - DataInputStream dis = new DataInputStream(fis); - dis.readFully(ser); - dis.close(); - fis.close(); - - Ast ast = p.parse(ser); - Ast ast2 = p.parse(ser); - assertEquals(ast.root, ast2.root); // TODO this fails :( - } - - public void Walk(INode node, StringBuilder sb) - { - if(node instanceof SequenceNode) - { - sb.append(String.format("%s (seq)\n", node.getClass())); - SequenceNode seq = (SequenceNode)node; - for(INode child: seq.elements) { - Walk(child, sb); - } - } - else - sb.append(String.format(" %s\n", node.toString())); - } -} +/** + * Serpent, a Python literal expression serializer/deserializer + * (a.k.a. Python's ast.literal_eval in Java) + * Software license: "MIT software license". See http://opensource.org/licenses/MIT + * @author Irmen de Jong (irmen@razorvine.net) + */ + +package net.razorvine.serpent.test; + +import static org.junit.Assert.*; + +import java.io.DataInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + +import net.razorvine.serpent.ObjectifyVisitor; +import net.razorvine.serpent.ParseException; +import net.razorvine.serpent.Parser; +import net.razorvine.serpent.SeekableStringReader; +import net.razorvine.serpent.Serializer; +import net.razorvine.serpent.ast.*; + +import org.junit.Ignore; +import org.junit.Test; + + +public class ParserTest +{ + @Test + public void TestBasic() + { + Parser p = new Parser(); + assertNull(p.parse((String)null).root); + assertNull(p.parse("").root); + assertNotNull(p.parse("# comment\n42\n").root); + } + + @Test + public void TestComments() + { + Parser p = new Parser(); + + Ast ast = p.parse("[ 1, 2 ]"); // no header whatsoever + ObjectifyVisitor visitor = new ObjectifyVisitor(); + ast.accept(visitor); + Object obj = visitor.getObject(); + + List expected = new ArrayList(); + expected.add(1); + expected.add(2); + assertEquals(expected, obj); + expected = new LinkedList(); + expected.add(1); + expected.add(2); + assertEquals(expected, obj); + + ast = p.parse("# serpent utf-8 python2.7\n"+ +"[ 1, 2,\n"+ +" # some comments here\n"+ +" 3, 4] # more here\n"+ +"# and here.\n"); + visitor = new ObjectifyVisitor(); + ast.accept(visitor); + obj = visitor.getObject(); + expected = new LinkedList(); + expected.add(1); + expected.add(2); + expected.add(3); + expected.add(4); + assertEquals(expected, obj); + } + + @Test + public void TestPrimitives() + { + Parser p = new Parser(); + assertEquals(new IntegerNode(42), p.parse("42").root); + assertEquals(new IntegerNode(-42), p.parse("-42").root); + assertEquals(new DoubleNode(42.331), p.parse("42.331").root); + assertEquals(new DoubleNode(-42.331), p.parse("-42.331").root); + assertEquals(new DoubleNode(-1.2e19), p.parse("-1.2e+19").root); + assertEquals(new DoubleNode(-1.2e-19), p.parse("-1.2e-19").root); + assertEquals(new DoubleNode(0.0004), p.parse("4e-4").root); + assertEquals(new DoubleNode(40000), p.parse("4e4").root); + assertEquals(new BooleanNode(true), p.parse("True").root); + assertEquals(new BooleanNode(false), p.parse("False").root); + assertEquals(NoneNode.Instance, p.parse("None").root); + + // long ints + assertEquals(new BigIntNode(new BigInteger("123456789123456789123456789")), p.parse("123456789123456789123456789").root); + assertFalse(new LongNode(52).equals(p.parse("52").root)); + assertEquals(new LongNode(123456789123456789L), p.parse("123456789123456789").root); + + assertEquals(BigIntNode.class, p.parse("12345678912345678912345678912345678978571892375798273578927389758378467693485903859038593453475897349587348957893457983475983475893475893785732957398475").root.getClass()); + } + + @Test + public void TestWeirdFloats() + { + Parser p = new Parser(); + TupleNode tuple = (TupleNode) p.parse("(1e30000,-1e30000,(1e30000+3.4j),{'__class__':'float','value':'nan'})").root; + assertEquals(4, tuple.elements.size()); + DoubleNode d = (DoubleNode) tuple.elements.get(0); + assertTrue(Double.isInfinite(d.value)); + d = (DoubleNode) tuple.elements.get(1); + assertTrue(Double.isInfinite(d.value)); + assertTrue(d.value < 0.0); + ComplexNumberNode c = (ComplexNumberNode) tuple.elements.get(2); + assertTrue(Double.isInfinite(c.real)); + assertEquals(3.4, c.imaginary, 0); + d = (DoubleNode) tuple.elements.get(3); + assertTrue(Double.isNaN(d.value)); + } + + @Test + public void TestFloatPrecision() + { + Parser p = new Parser(); + Serializer serpent = new Serializer(); + byte[] ser = serpent.serialize(1.2345678987654321); +System.out.println(new String(ser)); // TODO remove + DoubleNode dv = (DoubleNode) p.parse(ser).root; + assertEquals(new Double(1.2345678987654321), dv.value); + + ser = serpent.serialize(5555.12345678987656); +System.out.println(new String(ser)); // TODO remove + dv = (DoubleNode) p.parse(ser).root; + assertEquals(new Double(5555.12345678987656), dv.value); + + ser = serpent.serialize(98765432123456.12345678987656); +System.out.println(new String(ser)); // TODO remove + dv = (DoubleNode) p.parse(ser).root; + assertEquals(new Double(98765432123456.12345678987656), dv.value); + + ser = serpent.serialize(98765432123456.12345678987656e+44); + dv = (DoubleNode) p.parse(ser).root; +System.out.println(new String(ser)); // TODO remove + assertEquals(new Double(98765432123456.12345678987656e+44), dv.value); + + ComplexNumberNode cv = (ComplexNumberNode)p.parse("(98765432123456.12345678987656+665544332211.9998877665544j)").root; + assertEquals(new Double(98765432123456.12345678987656), cv.real, 0); + assertEquals(new Double(665544332211.9998877665544), cv.imaginary, 0); + cv = (ComplexNumberNode)p.parse("(98765432123456.12345678987656e+33+665544332211.9998877665544e+44j)").root; + assertEquals(new Double(98765432123456.12345678987656e+33), cv.real, 0); + assertEquals(new Double(665544332211.9998877665544e+44), cv.imaginary, 0); + } + + @Test + public void TestEquality() + { + INode n1, n2; + + n1 = new IntegerNode(42); + n2 = new IntegerNode(42); + assertEquals(n1, n2); + n2 = new IntegerNode(43); + assertFalse(n1.equals(n2)); + + n1 = new StringNode("foo"); + n2 = new StringNode("foo"); + assertEquals(n1, n2); + n2 = new StringNode("bar"); + assertFalse(n1.equals(n2)); + + ComplexNumberNode cn1 = new ComplexNumberNode(); + cn1.real=1.1; cn1.imaginary=2.2; + ComplexNumberNode cn2 = new ComplexNumberNode(); + cn2.real=1.1; cn2.imaginary=2.2; + assertEquals(cn1, cn2); + cn2 = new ComplexNumberNode(); + cn2.real=1.1; cn2.imaginary=3.3; + assertFalse(cn1.equals(cn2)); + + KeyValueNode kvn1=new KeyValueNode(); + kvn1.key = new IntegerNode(42); + kvn1.value = new IntegerNode(42); + KeyValueNode kvn2=new KeyValueNode(); + kvn2.key=new IntegerNode(42); + kvn2.value=new IntegerNode(42); + assertEquals(kvn1, kvn2); + kvn1=new KeyValueNode(); + kvn1.key=new IntegerNode(43); + kvn1.value=new IntegerNode(43); + assertFalse(kvn1.equals(kvn2)); + + n1=NoneNode.Instance; + n2=NoneNode.Instance; + assertEquals(n1, n2); + n2=new IntegerNode(42); + assertFalse(n1.equals(n2)); + + DictNode dn1 = new DictNode(); + kvn1 = new KeyValueNode(); + kvn1.key = new IntegerNode(42); + kvn1.value = new IntegerNode(42); + dn1.elements.add(kvn1); + DictNode dn2=new DictNode(); + kvn1 = new KeyValueNode(); + kvn1.key =new IntegerNode(42); + kvn1.value = new IntegerNode(42); + dn2.elements.add(kvn1); + assertEquals(dn1, dn2); + + dn2=new DictNode(); + kvn1 = new KeyValueNode(); + kvn1.key = new IntegerNode(42); + kvn1.value = new IntegerNode(43); + dn2.elements.add(kvn1); + assertFalse(dn1.equals(dn2)); + + ListNode ln1=new ListNode(); + ln1.elements.add(new IntegerNode(42)); + ListNode ln2=new ListNode(); + ln2.elements.add(new IntegerNode(42)); + assertEquals(ln1,ln2); + ln2=new ListNode(); + ln2.elements.add(new IntegerNode(43)); + assertFalse(ln1.equals(ln2)); + + SetNode sn1=new SetNode(); + sn1.elements.add(new IntegerNode(42)); + SetNode sn2=new SetNode(); + sn2.elements.add(new IntegerNode(42)); + assertEquals(sn1,sn2); + sn2=new SetNode(); + sn2.elements.add(new IntegerNode(43)); + assertFalse(sn1.equals(sn2)); + + TupleNode tn1=new TupleNode(); + tn1.elements.add(new IntegerNode(42)); + TupleNode tn2=new TupleNode(); + tn2.elements.add(new IntegerNode(42)); + assertEquals(tn1,tn2); + tn2=new TupleNode(); + tn2.elements.add(new IntegerNode(43)); + assertFalse(tn1.equals(tn2)); + + } + + @Test + public void TestPrintSingle() + { + Parser p = new Parser(); + + // primitives + assertEquals("42", p.parse("42").root.toString()); + assertEquals("-42.331", p.parse("-42.331").root.toString()); + assertEquals("-42.0", p.parse("-42.0").root.toString()); + assertEquals("-2.0E20", p.parse("-2E20").root.toString()); + assertEquals("2.0", p.parse("2.0").root.toString()); + assertEquals("1.2E19", p.parse("1.2e19").root.toString()); + assertEquals("True", p.parse("True").root.toString()); + assertEquals("'hello'", p.parse("'hello'").root.toString()); + assertEquals("'\\n'", p.parse("'\n'").root.toString()); + assertEquals("'\\''", p.parse("'\\''").root.toString()); + assertEquals("'\"'", p.parse("'\\\"'").root.toString()); + assertEquals("'\"'", p.parse("'\"'").root.toString()); + assertEquals("'\\\\'", p.parse("'\\\\'").root.toString()); + assertEquals("None", p.parse("None").root.toString()); + String ustr = "'\u20ac\u2603'"; + assertEquals(ustr, p.parse(ustr).root.toString()); + + // complex + assertEquals("(0.0+2.0j)", p.parse("2j").root.toString()); + assertEquals("(-1.1-2.2j)", p.parse("(-1.1-2.2j)").root.toString()); + assertEquals("(1.1+2.2j)", p.parse("(1.1+2.2j)").root.toString()); + + // long int + assertEquals("123456789123456789123456789", p.parse("123456789123456789123456789").root.toString()); + } + + @Test + public void TestPrintSeq() + { + Parser p=new Parser(); + + //tuple + assertEquals("()", p.parse("()").root.toString()); + assertEquals("(42,)", p.parse("(42,)").root.toString()); + assertEquals("(42,43)", p.parse("(42,43)").root.toString()); + + // list + assertEquals("[]", p.parse("[]").root.toString()); + assertEquals("[42]", p.parse("[42]").root.toString()); + assertEquals("[42,43]", p.parse("[42,43]").root.toString()); + + // set + assertEquals("{42}", p.parse("{42}").root.toString()); + assertEquals("{42,43}", p.parse("{42,43,43,43}").root.toString()); + + // dict + assertEquals("{}", p.parse("{}").root.toString()); + assertEquals("{'a':42}", p.parse("{'a': 42}").root.toString()); + + String result = p.parse("{'a': 42, 'b': 43}").root.toString(); + assertTrue(result.equals("{'a':42,'b':43}") || result.equals("{'b':43,'a':42}")); + result = p.parse("{'a': 42, 'b': 43, 'b': 44, 'b': 45}").root.toString(); + assertTrue(result.equals("{'a':42,'b':45}") || result.equals("{'b':45,'a':42}")); + } + + @Test(expected=ParseException.class) + public void TestInvalidPrimitive1() + { + new Parser().parse("1+2"); + } + + @Test(expected=ParseException.class) + public void TestInvalidPrimitive2() + { + new Parser().parse("1-2"); + } + + @Test(expected=ParseException.class) + public void TestInvalidPrimitive3() + { + new Parser().parse("1.1+2.2"); + } + + @Test(expected=ParseException.class) + public void TestInvalidPrimitive4() + { + new Parser().parse("1.1-2.2"); + } + + @Test(expected=ParseException.class) + public void TestInvalidPrimitive5() + { + new Parser().parse("True+2"); + } + + @Test(expected=ParseException.class) + public void TestInvalidPrimitive6() + { + new Parser().parse("False-2"); + } + + @Test(expected=ParseException.class) + public void TestInvalidPrimitive7() + { + new Parser().parse("3j+2"); + } + + @Test(expected=ParseException.class) + public void TestInvalidPrimitive8() + { + new Parser().parse("3j-2"); + } + + @Test(expected=ParseException.class) + public void TestInvalidPrimitive9() + { + new Parser().parse("None+2"); + } + + @Test(expected=ParseException.class) + public void TestInvalidPrimitive10() + { + new Parser().parse("None-2"); + } + + @Test(expected=ParseException.class) + public void TestInvalidTuple1() + { + new Parser().parse("(42,43]"); + } + + @Test(expected=ParseException.class) + public void TestInvalidTuple2() + { + new Parser().parse("()@"); + } + + @Test(expected=ParseException.class) + public void TestInvalidTuple3() + { + new Parser().parse("(42,43)@"); + } + + @Test(expected=ParseException.class) + public void TestInvalidList2() + { + new Parser().parse("[42,43}"); + } + + @Test(expected=ParseException.class) + public void TestInvalidList3() + { + new Parser().parse("[]@"); + } + + @Test(expected=ParseException.class) + public void TestInvalidList4() + { + new Parser().parse("[42,43]@"); + } + + @Test(expected=ParseException.class) + public void TestInvalidSet2() + { + new Parser().parse("{42,43]"); + } + + @Test(expected=ParseException.class) + public void TestInvalidSet3() + { + new Parser().parse("{42,43}@"); + } + + @Test(expected=ParseException.class) + public void TestInvalidDict2() + { + new Parser().parse("{'key': 42]"); + } + + @Test(expected=ParseException.class) + public void TestInvalidDict3() + { + new Parser().parse("{}@"); + } + + @Test(expected=ParseException.class) + public void TestInvalidDict4() + { + new Parser().parse("{'key': 42}@"); + } + + @Test + public void TestComplex() + { + Parser p = new Parser(); + ComplexNumberNode cplx = new ComplexNumberNode(); + cplx.real = 4.2; + cplx.imaginary = 3.2; + ComplexNumberNode cplx2 = new ComplexNumberNode(); + cplx2.real = 4.2; + cplx2.imaginary = 99; + assertFalse(cplx.equals(cplx2)); + cplx2.imaginary = 3.2; + assertEquals(cplx, cplx2); + + assertEquals(cplx, p.parse("(4.2+3.2j)").root); + cplx.real = 0; + assertEquals(cplx, p.parse("(0+3.2j)").root); + assertEquals(cplx, p.parse("3.2j").root); + assertEquals(cplx, p.parse("+3.2j").root); + cplx.imaginary = -3.2; + assertEquals(cplx, p.parse("-3.2j").root); + cplx.real = -9.9; + assertEquals(cplx, p.parse("(-9.9-3.2j)").root); + + cplx.real = 2; + cplx.imaginary = 3; + assertEquals(cplx, p.parse("(2+3j)").root); + cplx.imaginary = -3; + assertEquals(cplx, p.parse("(2-3j)").root); + cplx.real = 0; + assertEquals(cplx, p.parse("-3j").root); + + cplx.real = -3.2e32; + cplx.imaginary = -9.9e44; + assertEquals(cplx, p.parse("(-3.2e32 -9.9e44j)").root); + assertEquals(cplx, p.parse("(-3.2e+32 -9.9e+44j)").root); + assertEquals(cplx, p.parse("(-3.2e32-9.9e44j)").root); + assertEquals(cplx, p.parse("(-3.2e+32-9.9e+44j)").root); + cplx.imaginary = 9.9e44; + assertEquals(cplx, p.parse("(-3.2e32+9.9e44j)").root); + assertEquals(cplx, p.parse("(-3.2e+32+9.9e+44j)").root); + cplx.real = -3.2e-32; + cplx.imaginary = -9.9e-44; + assertEquals(cplx, p.parse("(-3.2e-32-9.9e-44j)").root); + } + + @Test + public void TestPrimitivesStuffAtEnd() + { + Parser p = new Parser(); + assertEquals(new IntegerNode(42), p.parseSingle(new SeekableStringReader("42@"))); + assertEquals(new DoubleNode(42.331), p.parseSingle(new SeekableStringReader("42.331@"))); + assertEquals(new BooleanNode(true), p.parseSingle(new SeekableStringReader("True@"))); + assertEquals(NoneNode.Instance, p.parseSingle(new SeekableStringReader("None@"))); + ComplexNumberNode cplx = new ComplexNumberNode(); + cplx.real=4; + cplx.imaginary=3; + assertEquals(cplx, p.parseSingle(new SeekableStringReader("(4+3j)@"))); + cplx.real=0; + assertEquals(cplx, p.parseSingle(new SeekableStringReader("3j@"))); + } + + @Test + public void TestStrings() + { + Parser p = new Parser(); + assertEquals(new StringNode("hello"), p.parse("'hello'").root); + assertEquals(new StringNode("hello"), p.parse("\"hello\"").root); + assertEquals(new StringNode("\\"), p.parse("'\\\\'").root); + assertEquals(new StringNode("\\"), p.parse("\"\\\\\"").root); + assertEquals(new StringNode("'"), p.parse("\"'\"").root); + assertEquals(new StringNode("\""), p.parse("'\"'").root); + assertEquals(new StringNode("tab\tnewline\n."), p.parse("'tab\\tnewline\\n.'").root); + } + + @Test + public void TestUnicode() throws UnsupportedEncodingException + { + Parser p = new Parser(); + String str = "'\u20ac\u2603'"; + assertEquals(0x20ac, str.charAt(1)); + assertEquals(0x2603, str.charAt(2)); + byte[] bytes = str.getBytes("utf-8"); + + String value = "\u20ac\u2603"; + assertEquals(new StringNode(value), p.parse(str).root); + assertEquals(new StringNode(value), p.parse(bytes).root); + } + + @Test + public void TestWhitespace() + { + Parser p = new Parser(); + assertEquals(new IntegerNode(42), p.parse(" 42 ").root); + assertEquals(new IntegerNode(42), p.parse(" 42 ").root); + assertEquals(new IntegerNode(42), p.parse("\t42\r\n").root); + assertEquals(new IntegerNode(42), p.parse(" \t 42 \r \n ").root); + assertEquals(new StringNode(" string value "), p.parse(" ' string value ' ").root); + try { + p.parse(" ( 42 , 43 , "); // missing tuple close ) + fail("expected parse error"); + } catch (ParseException x) { + //ok + } + + try { + p.parse(" [ 42 , 43 , "); // missing list close ) + fail("expected parse error"); + } catch (ParseException x) { + //ok + } + + try { + p.parse(" { 42 , 43 , "); // missing set close ) + fail("expected parse error"); + } catch (ParseException x) { + //ok + } + + try { + p.parse(" { 'a' : 4 2 , "); // missing dict close ) + fail("expected parse error"); + } catch (ParseException x) { + //ok + } + + Ast ast = p.parse(" ( 42 , ( 'x', 'y' ) ) "); + TupleNode tuple = (TupleNode) ast.root; + assertEquals(new IntegerNode(42), tuple.elements.get(0)); + tuple = (TupleNode) tuple.elements.get(1); + assertEquals(new StringNode("x"), tuple.elements.get(0)); + assertEquals(new StringNode("y"), tuple.elements.get(1)); + + p.parse(" ( 52 , ) "); + p.parse(" [ 52 ] "); + p.parse(" { 'a' : 42 } "); + p.parse(" { 52 } "); + } + + @Test + public void TestTuple() + { + Parser p = new Parser(); + TupleNode tuple = new TupleNode(); + TupleNode tuple2 = new TupleNode(); + assertEquals(tuple, tuple2); + + tuple.elements.add(new IntegerNode(42)); + tuple2.elements.add(new IntegerNode(99)); + assertFalse(tuple.equals(tuple2)); + tuple2.elements.clear(); + tuple2.elements.add(new IntegerNode(42)); + assertEquals(tuple, tuple2); + tuple2.elements.add(new IntegerNode(43)); + tuple2.elements.add(new IntegerNode(44)); + assertFalse(tuple.equals(tuple2)); + + assertEquals(new TupleNode(), p.parse("()").root); + assertEquals(tuple, p.parse("(42,)").root); + assertEquals(tuple2, p.parse("( 42,43, 44 )").root); + } + + @Test + public void TestList() + { + Parser p = new Parser(); + ListNode list = new ListNode(); + ListNode list2 = new ListNode(); + assertEquals(list, list2); + + list.elements.add(new IntegerNode(42)); + list2.elements.add(new IntegerNode(99)); + assertFalse(list.equals(list2)); + list2.elements.clear(); + list2.elements.add(new IntegerNode(42)); + assertEquals(list, list2); + list2.elements.add(new IntegerNode(43)); + list2.elements.add(new IntegerNode(44)); + assertFalse(list.equals(list2)); + + assertEquals(new ListNode(), p.parse("[]").root); + assertEquals(list, p.parse("[42]").root); + assertEquals(list2, p.parse("[ 42,43, 44 ]").root); + + } + + @Test + public void TestSet() + { + Parser p = new Parser(); + SetNode set1 = new SetNode(); + SetNode set2 = new SetNode(); + assertEquals(set1, set2); + + set1.elements.add(new IntegerNode(42)); + set2.elements.add(new IntegerNode(99)); + assertFalse(set1.equals(set2)); + set2.elements.clear(); + set2.elements.add(new IntegerNode(42)); + assertEquals(set1, set2); + + set2.elements.add(new IntegerNode(43)); + set2.elements.add(new IntegerNode(44)); + assertFalse(set1.equals(set2)); + + assertEquals(set1, p.parse("{42}").root); + assertEquals(set2, p.parse("{ 42,43, 44 }").root); + + set1 = (SetNode) p.parse("{'first','second','third','fourth','fifth','second', 'first', 'third', 'third' }").root; + assertTrue(set1.elements.contains(new StringNode("first"))); + assertTrue(set1.elements.contains(new StringNode("second"))); + assertTrue(set1.elements.contains(new StringNode("third"))); + assertTrue(set1.elements.contains(new StringNode("fourth"))); + assertTrue(set1.elements.contains(new StringNode("fifth"))); + assertEquals(5, set1.elements.size()); + } + + @Test + public void TestDict() + { + Parser p = new Parser(); + DictNode dict1 = new DictNode(); + DictNode dict2 = new DictNode(); + assertEquals(dict1, dict2); + + KeyValueNode kv1 = new KeyValueNode(); + kv1.key = new StringNode("key"); + kv1.value = new IntegerNode(42); + KeyValueNode kv2 = new KeyValueNode(); + kv2.key=new StringNode("key"); + kv2.value=new IntegerNode(99); + assertFalse(kv1.equals(kv2)); + kv2.value = new IntegerNode(42); + assertEquals(kv1, kv2); + + kv1=new KeyValueNode(); + kv1.key=new StringNode("key1"); + kv1.value=new IntegerNode(42); + dict1.elements.add(kv1); + kv1=new KeyValueNode(); + kv1.key=new StringNode("key1"); + kv1.value=new IntegerNode(99); + dict2.elements.add(kv1); + assertFalse(dict1.equals(dict2)); + + dict2.elements.clear(); + + kv1=new KeyValueNode(); + kv1.key=new StringNode("key1"); + kv1.value=new IntegerNode(42); + dict2.elements.add(kv1); + assertEquals(dict1, dict2); + + kv1=new KeyValueNode(); + kv1.key=new StringNode("key2"); + kv1.value=new IntegerNode(43); + dict2.elements.add(kv1); + kv1=new KeyValueNode(); + kv1.key=new StringNode("key3"); + kv1.value=new IntegerNode(44); + dict2.elements.add(kv1); + assertFalse(dict1.equals(dict2)); + + assertEquals(new DictNode(), p.parse("{}").root); + assertEquals(dict1, p.parse("{'key1': 42}").root); + assertEquals(dict2, p.parse("{'key1': 42, 'key2': 43, 'key3':44}").root); + + dict1 = (DictNode) p.parse("{'a': 1, 'b': 2, 'c': 3, 'c': 4, 'c': 5, 'c': 6}").root; + + kv1 = new KeyValueNode(); + kv1.key=new StringNode("c"); + kv1.value=new IntegerNode(6); + assertTrue(dict1.elements.contains(kv1)); + assertEquals(3, dict1.elements.size()); + } + + @Test + public void TestKeyValueEquality() + { + KeyValueNode kv1=new KeyValueNode(); + kv1.key=new StringNode("key1"); + kv1.value=new IntegerNode(42); + + KeyValueNode kv2=new KeyValueNode(); + kv2.key=new StringNode("key1"); + kv2.value=new IntegerNode(42); + + assertEquals(kv1, kv2); + kv2.value=new IntegerNode(43); + assertFalse(kv1.equals(kv2)); + } + + @Test + public void TestDictEquality() + { + DictNode dict1 = new DictNode(); + KeyValueNode kv=new KeyValueNode(); + kv.key=new StringNode("key1"); + kv.value=new IntegerNode(42); + dict1.elements.add(kv); + + DictNode dict2 = new DictNode(); + kv=new KeyValueNode(); + kv.key=new StringNode("key1"); + kv.value=new IntegerNode(42); + dict2.elements.add(kv); + + assertEquals(dict1, dict2); + + + kv=new KeyValueNode(); + kv.key=new StringNode("key2"); + kv.value=new IntegerNode(43); + dict1.elements.add(kv); + kv=new KeyValueNode(); + kv.key=new StringNode("key3"); + kv.value=new IntegerNode(44); + dict1.elements.add(kv); + + dict2 = new DictNode(); + kv=new KeyValueNode(); + kv.key=new StringNode("key2"); + kv.value=new IntegerNode(43); + dict2.elements.add(kv); + kv=new KeyValueNode(); + kv.key=new StringNode("key3"); + kv.value=new IntegerNode(44); + dict2.elements.add(kv); + kv=new KeyValueNode(); + kv.key=new StringNode("key1"); + kv.value=new IntegerNode(42); + dict2.elements.add(kv); + + assertTrue(dict1.equals(dict2)); + assertEquals(dict1, dict2); + kv=new KeyValueNode(); + kv.key=new StringNode("key4"); + kv.value=new IntegerNode(45); + dict2.elements.add(kv); + assertFalse(dict1.equals(dict2)); + } + + @Test + public void TestSetEquality() + { + SetNode set1 = new SetNode(); + set1.elements.add(new IntegerNode(1)); + set1.elements.add(new IntegerNode(2)); + set1.elements.add(new IntegerNode(3)); + + SetNode set2 = new SetNode(); + set2.elements.add(new IntegerNode(2)); + set2.elements.add(new IntegerNode(3)); + set2.elements.add(new IntegerNode(1)); + + assertEquals(set1, set2); + set2.elements.add(new IntegerNode(0)); + assertFalse(set1.equals(set2)); + } + + @Test + public void TestFile() throws IOException + { + Parser p = new Parser(); + File testdatafile = new File("src/test/java/testserpent.utf8.bin"); + byte[] ser = new byte[(int) testdatafile.length()]; + FileInputStream fis=new FileInputStream(testdatafile); + DataInputStream dis = new DataInputStream(fis); + dis.readFully(ser); + dis.close(); + fis.close(); + + Ast ast = p.parse(ser); + + String expr = ast.toString(); + Ast ast2 = p.parse(expr); + String expr2 = ast2.toString(); + assertEquals(expr.length(), expr2.length()); + + StringBuilder sb= new StringBuilder(); + Walk(ast.root, sb); + String walk1 = sb.toString(); + sb= new StringBuilder(); + Walk(ast2.root, sb); + String walk2 = sb.toString(); + assertEquals(walk1.length(), walk2.length()); + + // TODO assertEquals(ast.root, ast2.root); + ast = p.parse(expr2); + // TODO assertEquals(ast.root, ast2.root); + } + + @Test + public void TestTrailingCommas() throws IOException + { + Parser p = new Parser(); + INode result; + result = p.parse("[1,2,3, ]").root; + result = p.parse("[1,2,3 , ]").root; + result = p.parse("[1,2,3,]").root; + assertEquals("[1,2,3]", result.toString()); + result = p.parse("(1,2,3, )").root; + result = p.parse("(1,2,3 , )").root; + result = p.parse("(1,2,3,)").root; + assertEquals("(1,2,3)", result.toString()); + + // for dict and set the asserts are a bit more complex + // we cannot simply convert to string because the order of elts is undefined. + + result = p.parse("{'a':1, 'b':2, 'c':3, }").root; + result = p.parse("{'a':1, 'b':2, 'c':3 , }").root; + result = p.parse("{'a':1, 'b':2, 'c':3,}").root; + DictNode dict = (DictNode) result; + assertEquals(3, dict.elements.size()); + Set elts = dict.elementsAsSet(); + assertTrue(elts.contains(new KeyValueNode(new StringNode("a"), new IntegerNode(1)))); + assertTrue(elts.contains(new KeyValueNode(new StringNode("b"), new IntegerNode(2)))); + assertTrue(elts.contains(new KeyValueNode(new StringNode("c"), new IntegerNode(3)))); + result = p.parse("{1,2,3, }").root; + result = p.parse("{1,2,3 , }").root; + result = p.parse("{1,2,3,}").root; + SetNode set = (SetNode) result; + assertEquals(3, set.elements.size()); + elts = set.elementsAsSet(); + assertTrue(elts.contains(new IntegerNode(1))); + assertTrue(elts.contains(new IntegerNode(2))); + assertTrue(elts.contains(new IntegerNode(3))); + } + + + @Test + @Ignore("can't get the ast compare to succeed :(") + public void TestAstEquals() throws IOException + { + Parser p = new Parser(); + File testdatafile = new File("test/testserpent.utf8.bin"); + byte[] ser = new byte[(int) testdatafile.length()]; + FileInputStream fis=new FileInputStream(testdatafile); + DataInputStream dis = new DataInputStream(fis); + dis.readFully(ser); + dis.close(); + fis.close(); + + Ast ast = p.parse(ser); + Ast ast2 = p.parse(ser); + assertEquals(ast.root, ast2.root); // TODO this fails :( + } + + public void Walk(INode node, StringBuilder sb) + { + if(node instanceof SequenceNode) + { + sb.append(String.format("%s (seq)\n", node.getClass())); + SequenceNode seq = (SequenceNode)node; + for(INode child: seq.elements) { + Walk(child, sb); + } + } + else + sb.append(String.format(" %s\n", node.toString())); + } +} From 62cad4d56f94a462840e6c41da7afbbe0a2d07c9 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sat, 24 Dec 2016 18:58:29 +0100 Subject: [PATCH 042/230] nuget pack now fails if test fails --- dotnet/nuget-pack.cmd | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dotnet/nuget-pack.cmd b/dotnet/nuget-pack.cmd index 4e8f1c0..c0834fd 100644 --- a/dotnet/nuget-pack.cmd +++ b/dotnet/nuget-pack.cmd @@ -1,5 +1,7 @@ @echo Running tests... @call test-dotnet.cmd +if %errorlevel% neq 0 exit /b %errorlevel% + @echo. @echo Building NuGet package... nuget pack Serpent\Serpent.nuspec From 90638f5a602aebb999ad25d7ee66e262d79c538f Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Mon, 26 Dec 2016 12:29:19 +0100 Subject: [PATCH 043/230] better separation of parse error tests (java has no float precision errors but fails to parse complex sometimes, .net has both problems) --- dotnet/Serpent.Test/ParserTest.cs | 51 +++++++++++++++-- .../razorvine/serpent/test/ParserTest.java | 57 +++++++++++++++---- 2 files changed, 90 insertions(+), 18 deletions(-) diff --git a/dotnet/Serpent.Test/ParserTest.cs b/dotnet/Serpent.Test/ParserTest.cs index 020fefc..f7a5288 100644 --- a/dotnet/Serpent.Test/ParserTest.cs +++ b/dotnet/Serpent.Test/ParserTest.cs @@ -115,12 +115,10 @@ public void TestFloatPrecision() Console.WriteLine(Encoding.UTF8.GetString(ser)); // TODO remove Assert.AreEqual(98765432123456.12345678987656e+44, dv.Value); - Ast.ComplexNumberNode cv = (Ast.ComplexNumberNode)p.Parse("(98765432123456.12345678987656+665544332211.9998877665544j)").Root; - Assert.AreEqual(98765432123456.12345678987656, cv.Real); - Assert.AreEqual(665544332211.9998877665544, cv.Imaginary); - cv = (Ast.ComplexNumberNode)p.Parse("(98765432123456.12345678987656e+33+665544332211.9998877665544e+44j)").Root; - Assert.AreEqual(98765432123456.12345678987656e+33, cv.Real); - Assert.AreEqual(665544332211.9998877665544e+44, cv.Imaginary); + ser = serpent.Serialize(-98765432123456.12345678987656e-44); + dv = (Ast.DoubleNode) p.Parse(ser).Root; + Console.WriteLine(Encoding.UTF8.GetString(ser)); // TODO remove + Assert.AreEqual(-98765432123456.12345678987656e-44, dv.Value); } [Test] @@ -445,6 +443,47 @@ public void TestComplex() Assert.AreEqual(cplx, p.Parse("(-3.2e-32-9.9e-44j)").Root); } + [Test] + public void TestComplexPrecision() + { + Parser p = new Parser(); + Ast.ComplexNumberNode cv = (Ast.ComplexNumberNode)p.Parse("(98765432123456.12345678987656+665544332211.9998877665544j)").Root; + Assert.AreEqual(98765432123456.12345678987656, cv.Real); + Assert.AreEqual(665544332211.9998877665544, cv.Imaginary); + cv = (Ast.ComplexNumberNode)p.Parse("(98765432123456.12345678987656-665544332211.9998877665544j)").Root; + Assert.AreEqual(98765432123456.12345678987656, cv.Real); + Assert.AreEqual(-665544332211.9998877665544, cv.Imaginary); + cv = (Ast.ComplexNumberNode)p.Parse("(98765432123456.12345678987656e+33+665544332211.9998877665544e+44j)").Root; + Assert.AreEqual(98765432123456.12345678987656e+33, cv.Real); + Assert.AreEqual(665544332211.9998877665544e+44, cv.Imaginary); + cv = (Ast.ComplexNumberNode)p.Parse("(-98765432123456.12345678987656e+33-665544332211.9998877665544e+44j)").Root; + Assert.AreEqual(-98765432123456.12345678987656e+33, cv.Real); + Assert.AreEqual(-665544332211.9998877665544e+44, cv.Imaginary); + } + + + [Test] + public void CplxFail1_temporary() + { + Parser p = new Parser(); + Ast.ComplexNumberNode cplx = new Ast.ComplexNumberNode(); + cplx.Real = -3.2e32; + cplx.Imaginary = -9.9e44; + Assert.AreEqual(cplx, p.Parse("(-3.2e32-9.9e44j)").Root); + Assert.AreEqual(cplx, p.Parse("(-3.2e+32-9.9e+44j)").Root); + } + + [Test] + public void CplxFail2_temporary() + { + Parser p = new Parser(); + Ast.ComplexNumberNode cplx = new Ast.ComplexNumberNode(); + cplx.Real = -3.2e32; + cplx.Imaginary = -9.9e44; + Assert.AreEqual(cplx, p.Parse("(-3.2e32 -9.9e44j)").Root); + Assert.AreEqual(cplx, p.Parse("(-3.2e+32 -9.9e+44j)").Root); + } + [Test] public void TestPrimitivesStuffAtEnd() { diff --git a/java/src/test/java/net/razorvine/serpent/test/ParserTest.java b/java/src/test/java/net/razorvine/serpent/test/ParserTest.java index b63482d..d5e5882 100644 --- a/java/src/test/java/net/razorvine/serpent/test/ParserTest.java +++ b/java/src/test/java/net/razorvine/serpent/test/ParserTest.java @@ -125,31 +125,24 @@ public void TestFloatPrecision() Parser p = new Parser(); Serializer serpent = new Serializer(); byte[] ser = serpent.serialize(1.2345678987654321); -System.out.println(new String(ser)); // TODO remove DoubleNode dv = (DoubleNode) p.parse(ser).root; assertEquals(new Double(1.2345678987654321), dv.value); ser = serpent.serialize(5555.12345678987656); -System.out.println(new String(ser)); // TODO remove dv = (DoubleNode) p.parse(ser).root; assertEquals(new Double(5555.12345678987656), dv.value); ser = serpent.serialize(98765432123456.12345678987656); -System.out.println(new String(ser)); // TODO remove dv = (DoubleNode) p.parse(ser).root; assertEquals(new Double(98765432123456.12345678987656), dv.value); ser = serpent.serialize(98765432123456.12345678987656e+44); dv = (DoubleNode) p.parse(ser).root; -System.out.println(new String(ser)); // TODO remove assertEquals(new Double(98765432123456.12345678987656e+44), dv.value); - ComplexNumberNode cv = (ComplexNumberNode)p.parse("(98765432123456.12345678987656+665544332211.9998877665544j)").root; - assertEquals(new Double(98765432123456.12345678987656), cv.real, 0); - assertEquals(new Double(665544332211.9998877665544), cv.imaginary, 0); - cv = (ComplexNumberNode)p.parse("(98765432123456.12345678987656e+33+665544332211.9998877665544e+44j)").root; - assertEquals(new Double(98765432123456.12345678987656e+33), cv.real, 0); - assertEquals(new Double(665544332211.9998877665544e+44), cv.imaginary, 0); + ser = serpent.serialize(-98765432123456.12345678987656e-44); + dv = (DoubleNode) p.parse(ser).root; + assertEquals(new Double(-98765432123456.12345678987656e-44), dv.value); } @Test @@ -465,10 +458,10 @@ public void TestComplex() cplx.real = -3.2e32; cplx.imaginary = -9.9e44; - assertEquals(cplx, p.parse("(-3.2e32 -9.9e44j)").root); - assertEquals(cplx, p.parse("(-3.2e+32 -9.9e+44j)").root); assertEquals(cplx, p.parse("(-3.2e32-9.9e44j)").root); assertEquals(cplx, p.parse("(-3.2e+32-9.9e+44j)").root); + assertEquals(cplx, p.parse("(-3.2e32 -9.9e44j)").root); + assertEquals(cplx, p.parse("(-3.2e+32 -9.9e+44j)").root); cplx.imaginary = 9.9e44; assertEquals(cplx, p.parse("(-3.2e32+9.9e44j)").root); assertEquals(cplx, p.parse("(-3.2e+32+9.9e+44j)").root); @@ -476,7 +469,47 @@ public void TestComplex() cplx.imaginary = -9.9e-44; assertEquals(cplx, p.parse("(-3.2e-32-9.9e-44j)").root); } + + @Test + public void TestComplexPrecision() + { + Parser p = new Parser(); + ComplexNumberNode cv = (ComplexNumberNode)p.parse("(98765432123456.12345678987656+665544332211.9998877665544j)").root; + assertEquals(new Double(98765432123456.12345678987656), cv.real, 0); + assertEquals(new Double(665544332211.9998877665544), cv.imaginary, 0); + cv = (ComplexNumberNode)p.parse("(98765432123456.12345678987656-665544332211.9998877665544j)").root; + assertEquals(new Double(98765432123456.12345678987656), cv.real, 0); + assertEquals(new Double(-665544332211.9998877665544), cv.imaginary, 0); + cv = (ComplexNumberNode)p.parse("(98765432123456.12345678987656e+33+665544332211.9998877665544e+44j)").root; + assertEquals(new Double(98765432123456.12345678987656e+33), cv.real, 0); + assertEquals(new Double(665544332211.9998877665544e+44), cv.imaginary, 0); + cv = (ComplexNumberNode)p.parse("(-98765432123456.12345678987656e+33-665544332211.9998877665544e+44j)").root; + assertEquals(new Double(-98765432123456.12345678987656e+33), cv.real, 0); + assertEquals(new Double(-665544332211.9998877665544e+44), cv.imaginary, 0); + } + + @Test + public void CplxFail1_temporary() + { + Parser p = new Parser(); + ComplexNumberNode cplx = new ComplexNumberNode(); + cplx.real = -3.2e32; + cplx.imaginary = -9.9e44; + assertEquals(cplx, p.parse("(-3.2e32-9.9e44j)").root); + assertEquals(cplx, p.parse("(-3.2e+32-9.9e+44j)").root); + } + @Test + public void CplxFail2_temporary() + { + Parser p = new Parser(); + ComplexNumberNode cplx = new ComplexNumberNode(); + cplx.real = -3.2e32; + cplx.imaginary = -9.9e44; + assertEquals(cplx, p.parse("(-3.2e32 -9.9e44j)").root); + assertEquals(cplx, p.parse("(-3.2e+32 -9.9e+44j)").root); + } + @Test public void TestPrimitivesStuffAtEnd() { From 935fc6e418ea9ce201d3855d70afebf52c1b3703 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Mon, 26 Dec 2016 14:02:49 +0100 Subject: [PATCH 044/230] can now serialize enum types (python 3.4+) fixes #20 --- README.md | 1 + serpent.py | 7 +++++++ test_serpent.py | 11 ++++++++--- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5fae2ce..b7498b8 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ Serpent handles several special Python types to make life easier: - array.array other typecode --> list - Exception --> dict with some fields of the exception (message, args) - collections module types --> mostly equivalent primitive types or dict + - enums --> the value of the enum (Python 3.4+) - all other types --> dict with the ``__getstate__`` or ``vars()`` of the object Notes: diff --git a/serpent.py b/serpent.py index 20cf8c8..1540ed9 100644 --- a/serpent.py +++ b/serpent.py @@ -19,6 +19,8 @@ - array.array typecode 'c'/'u' --> string/unicode - array.array other typecode --> list - Exception --> dict with some fields of the exception (message, args) + - collections module types --> mostly equivalent primitive types or dict + - enums --> the value of the enum (Python 3.4+) - all other types --> dict with __getstate__ or vars() of the object Notes: @@ -126,6 +128,11 @@ def _ser_DictView(obj, serializer, outputstream, indentlevel): } if sys.version_info >= (2, 7): _special_classes_registry[collections.OrderedDict] = _ser_OrderedDict +if sys.version_info >= (3, 4): + import enum + def _ser_Enum(obj, serializer, outputstream, indentlevel): + serializer._serialize(obj.value, outputstream, indentlevel) + _special_classes_registry[enum.Enum] = _ser_Enum def unregister_class(clazz): diff --git a/test_serpent.py b/test_serpent.py index c36d07b..167cccc 100644 --- a/test_serpent.py +++ b/test_serpent.py @@ -562,6 +562,8 @@ def test_float_precision(self): self.assertEqual(98765432123456.12345678987656e+44, v) v = serpent.loads(serpent.dumps((98765432123456.12345678987656e+44+665544332211.9998877665544e+33j))) self.assertEqual((98765432123456.12345678987656e+44+665544332211.9998877665544e+33j), v) + v = serpent.loads(serpent.dumps((-98765432123456.12345678987656e+44 -665544332211.9998877665544e+33j))) + self.assertEqual((-98765432123456.12345678987656e+44 -665544332211.9998877665544e+33j), v) @unittest.skipIf(sys.version_info < (3, 4), "needs python 3.4 to test enum type") def test_enums(self): @@ -570,9 +572,11 @@ class Animal(enum.Enum): BEE = 1 CAT = 2 DOG = 3 - print(Animal.CAT) v = serpent.loads(serpent.dumps(Animal.CAT)) - self.assertEqual("Animal.CAT", v) + self.assertEqual(2, v) + Animal2 = enum.Enum("Animals2", "BEE CAT DOG HORSE RABBIT") + v = serpent.loads(serpent.dumps(Animal2.HORSE)) + self.assertEqual(4, v) def test_tobytes(self): obj = b"test" @@ -803,7 +807,8 @@ def custom_serializer(obj, serializer, stream, level): s = SubClass() d = serpent.dumps(s) x = serpent.loads(d) - self.assertEqual("[(sub)class=]", x) + classname = __name__+".SubClass" + self.assertEqual("[(sub)class=]", x) def testUUID(self): uid = uuid.uuid4() From cde3940596a438ab833cb5298e40b56d2a2b1f0c Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 27 Dec 2016 21:26:50 +0100 Subject: [PATCH 045/230] fix float precision tests --- dotnet/Serpent.Test/ParserTest.cs | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/dotnet/Serpent.Test/ParserTest.cs b/dotnet/Serpent.Test/ParserTest.cs index f7a5288..7ca7da2 100644 --- a/dotnet/Serpent.Test/ParserTest.cs +++ b/dotnet/Serpent.Test/ParserTest.cs @@ -96,29 +96,24 @@ public void TestFloatPrecision() Parser p = new Parser(); Serializer serpent = new Serializer(); var ser = serpent.Serialize(1.2345678987654321); - Console.WriteLine(Encoding.UTF8.GetString(ser)); // TODO remove Ast.DoubleNode dv = (Ast.DoubleNode) p.Parse(ser).Root; - Assert.AreEqual(1.2345678987654321, dv.Value); + Assert.AreEqual(1.2345678987654321.ToString(), dv.Value.ToString()); ser = serpent.Serialize(5555.12345678987656); - Console.WriteLine(Encoding.UTF8.GetString(ser)); // TODO remove dv = (Ast.DoubleNode) p.Parse(ser).Root; - Assert.AreEqual(5555.12345678987656, dv.Value); + Assert.AreEqual(5555.12345678987656.ToString(), dv.Value.ToString()); ser = serpent.Serialize(98765432123456.12345678987656); - Console.WriteLine(Encoding.UTF8.GetString(ser)); // TODO remove dv = (Ast.DoubleNode) p.Parse(ser).Root; - Assert.AreEqual(98765432123456.12345678987656, dv.Value); + Assert.AreEqual(98765432123456.12345678987656.ToString(), dv.Value.ToString()); ser = serpent.Serialize(98765432123456.12345678987656e+44); dv = (Ast.DoubleNode) p.Parse(ser).Root; - Console.WriteLine(Encoding.UTF8.GetString(ser)); // TODO remove - Assert.AreEqual(98765432123456.12345678987656e+44, dv.Value); + Assert.AreEqual((98765432123456.12345678987656e+44).ToString(), dv.Value.ToString()); ser = serpent.Serialize(-98765432123456.12345678987656e-44); dv = (Ast.DoubleNode) p.Parse(ser).Root; - Console.WriteLine(Encoding.UTF8.GetString(ser)); // TODO remove - Assert.AreEqual(-98765432123456.12345678987656e-44, dv.Value); + Assert.AreEqual((-98765432123456.12345678987656e-44).ToString(), dv.Value.ToString()); } [Test] From 351a95e8c5fc68c4bcb596bf8213d999ffeb8896 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 27 Dec 2016 21:44:04 +0100 Subject: [PATCH 046/230] improve parsing of complex vs tuple --- dotnet/Serpent/Parser.cs | 13 +++++-------- .../src/main/java/net/razorvine/serpent/Parser.java | 13 +++++-------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/dotnet/Serpent/Parser.cs b/dotnet/Serpent/Parser.cs index 4a85379..6ebfef4 100644 --- a/dotnet/Serpent/Parser.cs +++ b/dotnet/Serpent/Parser.cs @@ -97,16 +97,13 @@ Ast.INode ParseCompound(SeekableStringReader sr) } } case '(': - // tricky case here, it can be a tuple but also a complex number. - // try complex number first + // tricky case here, it can be a tuple but also a complex number: + // if the last character before the closing parenthesis is a 'j', it is a complex number { int bm = sr.Bookmark(); - try { - return ParseComplex(sr); - } catch(ParseException) { - sr.FlipBack(bm); - return ParseTuple(sr); - } + string betweenparens = sr.ReadUntil(')').TrimEnd(); + sr.FlipBack(bm); + return betweenparens.EndsWith("j") ? (Ast.INode) ParseComplex(sr) : ParseTuple(sr); } default: throw new ParseException("invalid sequencetype char"); diff --git a/java/src/main/java/net/razorvine/serpent/Parser.java b/java/src/main/java/net/razorvine/serpent/Parser.java index 0783e86..1344e60 100644 --- a/java/src/main/java/net/razorvine/serpent/Parser.java +++ b/java/src/main/java/net/razorvine/serpent/Parser.java @@ -104,16 +104,13 @@ INode parseCompound(SeekableStringReader sr) } } case '(': - // tricky case here, it can be a tuple but also a complex number. - // try complex number first + // tricky case here, it can be a tuple but also a complex number: + // if the last character before the closing parenthesis is a 'j', it is a complex number { int bm = sr.bookmark(); - try { - return parseComplex(sr); - } catch(ParseException x) { - sr.flipBack(bm); - return parseTuple(sr); - } + String betweenparens = sr.readUntil(')').trim(); + sr.flipBack(bm); + return betweenparens.endsWith("j") ? parseComplex(sr) : parseTuple(sr); } default: throw new ParseException("invalid sequencetype char"); From 10c5b353de5ac91bac86e73ff55762a9ae316fa2 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 27 Dec 2016 22:49:35 +0100 Subject: [PATCH 047/230] fixed complex number parsing errors, tests --- dotnet/Serpent.Test/ParserTest.cs | 20 ++++++++++------ dotnet/Serpent/Parser.cs | 18 ++++++++++++-- .../java/net/razorvine/serpent/Parser.java | 18 ++++++++++++-- .../razorvine/serpent/test/ParserTest.java | 24 +++++++++++++------ 4 files changed, 62 insertions(+), 18 deletions(-) diff --git a/dotnet/Serpent.Test/ParserTest.cs b/dotnet/Serpent.Test/ParserTest.cs index 7ca7da2..3889ace 100644 --- a/dotnet/Serpent.Test/ParserTest.cs +++ b/dotnet/Serpent.Test/ParserTest.cs @@ -77,17 +77,23 @@ public void TestPrimitives() public void TestWeirdFloats() { Parser p = new Parser(); - var tuple = (Ast.TupleNode) p.Parse("(1e30000,-1e30000,(1e30000+3.4j),{'__class__':'float','value':'nan'})").Root; - Assert.AreEqual(4, tuple.Elements.Count); - var d = (Ast.DoubleNode) tuple.Elements[0]; + var d = (Ast.DoubleNode) p.Parse("1e30000").Root; + Assert.IsTrue(double.IsPositiveInfinity(d.Value)); + d = (Ast.DoubleNode) p.Parse("-1e30000").Root; + Assert.IsTrue(double.IsNegativeInfinity(d.Value)); + + var tuple = (Ast.TupleNode) p.Parse("(1e30000,-1e30000,{'__class__':'float','value':'nan'})").Root; + Assert.AreEqual(3, tuple.Elements.Count); + d = (Ast.DoubleNode) tuple.Elements[0]; Assert.IsTrue(double.IsPositiveInfinity(d.Value)); d = (Ast.DoubleNode) tuple.Elements[1]; Assert.IsTrue(double.IsNegativeInfinity(d.Value)); - var c = (Ast.ComplexNumberNode) tuple.Elements[2]; - Assert.IsTrue(double.IsInfinity(c.Real)); - Assert.AreEqual(3.4, c.Imaginary, 0); - d = (Ast.DoubleNode) tuple.Elements[3]; + d = (Ast.DoubleNode) tuple.Elements[2]; Assert.IsTrue(Double.IsNaN(d.Value)); + + var c = (Ast.ComplexNumberNode) p.Parse("(1e30000-1e30000j)").Root; + Assert.IsTrue(double.IsPositiveInfinity(c.Real)); + Assert.IsTrue(double.IsNegativeInfinity(c.Imaginary)); } [Test] diff --git a/dotnet/Serpent/Parser.cs b/dotnet/Serpent/Parser.cs index 6ebfef4..98673f9 100644 --- a/dotnet/Serpent/Parser.cs +++ b/dotnet/Serpent/Parser.cs @@ -101,7 +101,7 @@ Ast.INode ParseCompound(SeekableStringReader sr) // if the last character before the closing parenthesis is a 'j', it is a complex number { int bm = sr.Bookmark(); - string betweenparens = sr.ReadUntil(')').TrimEnd(); + string betweenparens = sr.ReadUntil(new char[]{')','\n'}).TrimEnd(); sr.FlipBack(bm); return betweenparens.EndsWith("j") ? (Ast.INode) ParseComplex(sr) : ParseTuple(sr); } @@ -428,14 +428,28 @@ Ast.ComplexNumberNode ParseComplex(SeekableStringReader sr) numberstr = sr.Read(1) + sr.ReadUntil(new char[] {'+', '-'}); } else + { numberstr = sr.ReadUntil(new char[] {'+', '-'}); + } + sr.Rewind(1); // rewind the +/- + + // because we're a bit more cautious here with reading chars than in the float parser, + // it can be that the parser now stopped directly after the 'e' in a number like "3.14e+20". + // ("3.14e20" is fine) So, check if the last char is 'e' and if so, continue reading 0..9. + if(numberstr.EndsWith("e", StringComparison.InvariantCultureIgnoreCase)) { + // if the next symbol is + or -, accept it, then read the exponent integer + if(sr.Peek()=='-' || sr.Peek()=='+') + numberstr+=sr.Read(1); + numberstr += sr.ReadWhile("0123456789"); + } + + sr.SkipWhitespace(); double realpart; try { realpart = this.ParseDouble(numberstr); } catch (FormatException x) { throw new ParseException("invalid float format", x); } - sr.Rewind(1); // rewind the +/- double imaginarypart = ParseImaginaryPart(sr); if(sr.Read()!=')') throw new ParseException("expected ) to end a complex number"); diff --git a/java/src/main/java/net/razorvine/serpent/Parser.java b/java/src/main/java/net/razorvine/serpent/Parser.java index 1344e60..32c1597 100644 --- a/java/src/main/java/net/razorvine/serpent/Parser.java +++ b/java/src/main/java/net/razorvine/serpent/Parser.java @@ -108,7 +108,7 @@ INode parseCompound(SeekableStringReader sr) // if the last character before the closing parenthesis is a 'j', it is a complex number { int bm = sr.bookmark(); - String betweenparens = sr.readUntil(')').trim(); + String betweenparens = sr.readUntil(")\n").trim(); sr.flipBack(bm); return betweenparens.endsWith("j") ? parseComplex(sr) : parseTuple(sr); } @@ -433,14 +433,28 @@ ComplexNumberNode parseComplex(SeekableStringReader sr) numberstr = sr.read(1) + sr.readUntil("+-"); } else + { numberstr = sr.readUntil("+-"); + } + sr.rewind(1); // rewind the +/- + + // because we're a bit more cautious here with reading chars than in the float parser, + // it can be that the parser now stopped directly after the 'e' in a number like "3.14e+20". + // ("3.14e20" is fine) So, check if the last char is 'e' and if so, continue reading 0..9. + if(numberstr.endsWith("e")||numberstr.endsWith("E")) { + // if the next symbol is + or -, accept it, then read the exponent integer + if(sr.peek()=='-' || sr.peek()=='+') + numberstr+=sr.read(1); + numberstr += sr.readWhile("0123456789"); + } + + sr.skipWhitespace(); double real; try { real = Double.parseDouble(numberstr); } catch (NumberFormatException x) { throw new ParseException("invalid float format", x); } - sr.rewind(1); // rewind the +/- double imaginarypart = parseImaginaryPart(sr); if(sr.read()!=')') throw new ParseException("expected ) to end a complex number"); diff --git a/java/src/test/java/net/razorvine/serpent/test/ParserTest.java b/java/src/test/java/net/razorvine/serpent/test/ParserTest.java index d5e5882..c4462cb 100644 --- a/java/src/test/java/net/razorvine/serpent/test/ParserTest.java +++ b/java/src/test/java/net/razorvine/serpent/test/ParserTest.java @@ -105,18 +105,28 @@ public void TestPrimitives() public void TestWeirdFloats() { Parser p = new Parser(); - TupleNode tuple = (TupleNode) p.parse("(1e30000,-1e30000,(1e30000+3.4j),{'__class__':'float','value':'nan'})").root; - assertEquals(4, tuple.elements.size()); - DoubleNode d = (DoubleNode) tuple.elements.get(0); + DoubleNode d = (DoubleNode) p.parse("1e30000").root; + assertTrue(Double.isInfinite(d.value)); + assertTrue(d.value > 0.0); + d = (DoubleNode) p.parse("-1e30000").root; + assertTrue(Double.isInfinite(d.value)); + assertTrue(d.value < 0.0); + + TupleNode tuple = (TupleNode) p.parse("(1e30000,-1e30000,{'__class__':'float','value':'nan'})").root; + assertEquals(3, tuple.elements.size()); + d = (DoubleNode) tuple.elements.get(0); assertTrue(Double.isInfinite(d.value)); d = (DoubleNode) tuple.elements.get(1); assertTrue(Double.isInfinite(d.value)); assertTrue(d.value < 0.0); - ComplexNumberNode c = (ComplexNumberNode) tuple.elements.get(2); - assertTrue(Double.isInfinite(c.real)); - assertEquals(3.4, c.imaginary, 0); - d = (DoubleNode) tuple.elements.get(3); + d = (DoubleNode) tuple.elements.get(2); assertTrue(Double.isNaN(d.value)); + + ComplexNumberNode c = (ComplexNumberNode) p.parse("(1e30000-1e30000j)").root; + assertTrue(Double.isInfinite(c.real)); + assertTrue(c.real>0.0); + assertTrue(Double.isInfinite(c.imaginary)); + assertTrue(c.imaginary<0.0); } @Test From 7f47aaead8adb3c9a216fbe45d0a5a48013baf1d Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 27 Dec 2016 22:57:43 +0100 Subject: [PATCH 048/230] removed extra tests for complex number issues --- dotnet/Serpent.Test/ParserTest.cs | 25 +------------------ .../razorvine/serpent/test/ParserTest.java | 22 ---------------- 2 files changed, 1 insertion(+), 46 deletions(-) diff --git a/dotnet/Serpent.Test/ParserTest.cs b/dotnet/Serpent.Test/ParserTest.cs index 3889ace..6596b89 100644 --- a/dotnet/Serpent.Test/ParserTest.cs +++ b/dotnet/Serpent.Test/ParserTest.cs @@ -461,30 +461,7 @@ public void TestComplexPrecision() Assert.AreEqual(-98765432123456.12345678987656e+33, cv.Real); Assert.AreEqual(-665544332211.9998877665544e+44, cv.Imaginary); } - - - [Test] - public void CplxFail1_temporary() - { - Parser p = new Parser(); - Ast.ComplexNumberNode cplx = new Ast.ComplexNumberNode(); - cplx.Real = -3.2e32; - cplx.Imaginary = -9.9e44; - Assert.AreEqual(cplx, p.Parse("(-3.2e32-9.9e44j)").Root); - Assert.AreEqual(cplx, p.Parse("(-3.2e+32-9.9e+44j)").Root); - } - - [Test] - public void CplxFail2_temporary() - { - Parser p = new Parser(); - Ast.ComplexNumberNode cplx = new Ast.ComplexNumberNode(); - cplx.Real = -3.2e32; - cplx.Imaginary = -9.9e44; - Assert.AreEqual(cplx, p.Parse("(-3.2e32 -9.9e44j)").Root); - Assert.AreEqual(cplx, p.Parse("(-3.2e+32 -9.9e+44j)").Root); - } - + [Test] public void TestPrimitivesStuffAtEnd() { diff --git a/java/src/test/java/net/razorvine/serpent/test/ParserTest.java b/java/src/test/java/net/razorvine/serpent/test/ParserTest.java index c4462cb..493bfe0 100644 --- a/java/src/test/java/net/razorvine/serpent/test/ParserTest.java +++ b/java/src/test/java/net/razorvine/serpent/test/ParserTest.java @@ -498,28 +498,6 @@ public void TestComplexPrecision() assertEquals(new Double(-665544332211.9998877665544e+44), cv.imaginary, 0); } - @Test - public void CplxFail1_temporary() - { - Parser p = new Parser(); - ComplexNumberNode cplx = new ComplexNumberNode(); - cplx.real = -3.2e32; - cplx.imaginary = -9.9e44; - assertEquals(cplx, p.parse("(-3.2e32-9.9e44j)").root); - assertEquals(cplx, p.parse("(-3.2e+32-9.9e+44j)").root); - } - - @Test - public void CplxFail2_temporary() - { - Parser p = new Parser(); - ComplexNumberNode cplx = new ComplexNumberNode(); - cplx.real = -3.2e32; - cplx.imaginary = -9.9e44; - assertEquals(cplx, p.parse("(-3.2e32 -9.9e44j)").root); - assertEquals(cplx, p.parse("(-3.2e+32 -9.9e+44j)").root); - } - @Test public void TestPrimitivesStuffAtEnd() { From e3a425912a928a422155e717c06dad20acd56131 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 27 Dec 2016 23:00:32 +0100 Subject: [PATCH 049/230] [maven-release-plugin] prepare release serpent-1.16 --- java/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index e68dfe3..086e165 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -9,7 +9,7 @@ net.razorvine serpent - 1.16-SNAPSHOT + 1.16 jar serpent @@ -67,7 +67,7 @@ It is an alternative to JSON to provide easy data integration between Java and P https://github.com/irmen/Serpent scm:git:https://github.com/irmen/Serpent.git scm:git:https://github.com/irmen/Serpent.git - HEAD + serpent-1.16 Github From 15fb43d67547f54c54a57801cbc94169ac3480af Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 27 Dec 2016 23:00:40 +0100 Subject: [PATCH 050/230] [maven-release-plugin] prepare for next development iteration --- java/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 086e165..f5a0c09 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -9,7 +9,7 @@ net.razorvine serpent - 1.16 + 1.17-SNAPSHOT jar serpent @@ -67,7 +67,7 @@ It is an alternative to JSON to provide easy data integration between Java and P https://github.com/irmen/Serpent scm:git:https://github.com/irmen/Serpent.git scm:git:https://github.com/irmen/Serpent.git - serpent-1.16 + HEAD Github From 13b72c2c7ed134bc8659cd1dfd53933307fcce11 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 27 Dec 2016 23:14:52 +0100 Subject: [PATCH 051/230] nuspec releasenotes --- dotnet/Serpent/Serpent.nuspec | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dotnet/Serpent/Serpent.nuspec b/dotnet/Serpent/Serpent.nuspec index 0aa0684..5c53595 100644 --- a/dotnet/Serpent/Serpent.nuspec +++ b/dotnet/Serpent/Serpent.nuspec @@ -19,8 +19,7 @@ such a Python expression itself, back into the equivalent .NET datatypes. More info for the Python version is on Pypi: https://pypi.python.org/pypi/serpent Serpent Python literal expression serialization -Framework compatibility improvements. -ToBytes helper function can now deal with more dictionary types that are essentially the same. +Critical fix: fixed errors in complex number parser, it could crash on complex numbers containing float parts Copyright 2016 serialization python pyro From 847c3260e37c2bd50db6c86a4a07e0f64e60146b Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 27 Dec 2016 23:16:27 +0100 Subject: [PATCH 052/230] fix pom version --- java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/pom.xml b/java/pom.xml index f5a0c09..e68dfe3 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -9,7 +9,7 @@ net.razorvine serpent - 1.17-SNAPSHOT + 1.16-SNAPSHOT jar serpent From 354b58403f3cc1976dac73291f8bd3f58676a087 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 27 Dec 2016 23:19:12 +0100 Subject: [PATCH 053/230] [maven-release-plugin] prepare release serpent-1.16 --- java/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index e68dfe3..086e165 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -9,7 +9,7 @@ net.razorvine serpent - 1.16-SNAPSHOT + 1.16 jar serpent @@ -67,7 +67,7 @@ It is an alternative to JSON to provide easy data integration between Java and P https://github.com/irmen/Serpent scm:git:https://github.com/irmen/Serpent.git scm:git:https://github.com/irmen/Serpent.git - HEAD + serpent-1.16 Github From 4112c032fe6887bb38fa815a4f0baf707dc0b3dd Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 27 Dec 2016 23:19:20 +0100 Subject: [PATCH 054/230] [maven-release-plugin] prepare for next development iteration --- java/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 086e165..f5a0c09 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -9,7 +9,7 @@ net.razorvine serpent - 1.16 + 1.17-SNAPSHOT jar serpent @@ -67,7 +67,7 @@ It is an alternative to JSON to provide easy data integration between Java and P https://github.com/irmen/Serpent scm:git:https://github.com/irmen/Serpent.git scm:git:https://github.com/irmen/Serpent.git - serpent-1.16 + HEAD Github From 49e4d0b7f9254b9fa1cf839ab7326aa06327c228 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 27 Dec 2016 23:31:46 +0100 Subject: [PATCH 055/230] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index b7498b8..281be17 100644 --- a/README.md +++ b/README.md @@ -27,12 +27,14 @@ Example usage can be found in ./example.py C#/.NET ------- +Package is available on www.nuget.org as 'Razorvine.Serpent'. Full source code can be found in ./dotnet/ directory. Example usage can be found in ./dotnet/Serpent.Test/Example.cs JAVA ---- +Maven-artefact is available on maven central, groupid 'net.razorvine' artifactid 'serpent'. Full source code can be found in ./java/ directory. Example usage can be found in ./java/test/SerpentExample.java From df50f3bf1a7a217c6a9f508a66da3d9cfd718882 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Wed, 28 Dec 2016 12:39:13 +0100 Subject: [PATCH 056/230] changed .net framework version once again to be compatible with mono once more (4.5) --- dotnet/Serpent.Test/Serpent.Test.csproj | 8 +++----- dotnet/Serpent.Test/app.config | 5 +---- dotnet/Serpent/Serpent.csproj | 8 +++----- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/dotnet/Serpent.Test/Serpent.Test.csproj b/dotnet/Serpent.Test/Serpent.Test.csproj index 3fd7bc3..5d2c05f 100644 --- a/dotnet/Serpent.Test/Serpent.Test.csproj +++ b/dotnet/Serpent.Test/Serpent.Test.csproj @@ -13,11 +13,9 @@ False 4 false - 9.0.21022 + 8.0.30703 2.0 - Client - v4.0 - {FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} + v4.5 x86 @@ -80,4 +78,4 @@ - \ No newline at end of file + diff --git a/dotnet/Serpent.Test/app.config b/dotnet/Serpent.Test/app.config index 0d69660..a036389 100644 --- a/dotnet/Serpent.Test/app.config +++ b/dotnet/Serpent.Test/app.config @@ -1,6 +1,3 @@  - - - - \ No newline at end of file + diff --git a/dotnet/Serpent/Serpent.csproj b/dotnet/Serpent/Serpent.csproj index 24ba936..ad954e3 100644 --- a/dotnet/Serpent/Serpent.csproj +++ b/dotnet/Serpent/Serpent.csproj @@ -13,11 +13,9 @@ 4 false C:\Users\Irmen\AppData\Roaming\ICSharpCode/SharpDevelop4\Settings.SourceAnalysis - 10.0.0 + 8.0.30703 2.0 - Client - v4.0 - {FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} + v4.5 x86 @@ -71,4 +69,4 @@ - \ No newline at end of file + From 503d81f657effe370973cf581346d604af675925 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sat, 11 Feb 2017 16:00:45 +0100 Subject: [PATCH 057/230] added maven and nuget version badgets --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 281be17..127c851 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,8 @@ Serpent serialization library (Python/.NET/Java) [![Build Status](https://travis-ci.org/irmen/Serpent.svg?branch=master)](https://travis-ci.org/irmen/Serpent) [![Latest Version](https://img.shields.io/pypi/v/Serpent.svg)](https://pypi.python.org/pypi/Serpent/) - +[![Maven Central](https://img.shields.io/maven-central/v/net.razorvine/serpent.svg)](http://search.maven.org/#search|ga|1|g%3A%22net.razorvine%22%20AND%20a%3A%22serpent%22) +[![NuGet](https://img.shields.io/nuget/v/Razorvine.Serpent.svg)](https://www.nuget.org/packages/Razorvine.Serpent/) Serpent provides ast.literal_eval() compatible object tree serialization. It serializes an object tree into bytes (utf-8 encoded string) that can be decoded and then From ca01423586872bd02d55d0fb496cb97810ab9876 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sat, 11 Feb 2017 16:05:00 +0100 Subject: [PATCH 058/230] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 127c851..ee1e72f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ Serpent serialization library (Python/.NET/Java) ================================================ +[![saythanks](https://img.shields.io/badge/say-thanks-ff69b4.svg)](https://saythanks.io/to/irmen) [![Build Status](https://travis-ci.org/irmen/Serpent.svg?branch=master)](https://travis-ci.org/irmen/Serpent) [![Latest Version](https://img.shields.io/pypi/v/Serpent.svg)](https://pypi.python.org/pypi/Serpent/) [![Maven Central](https://img.shields.io/maven-central/v/net.razorvine/serpent.svg)](http://search.maven.org/#search|ga|1|g%3A%22net.razorvine%22%20AND%20a%3A%22serpent%22) From c3ba209d845d9439f4b62c452fdf2ffe9fde16e6 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Mon, 13 Feb 2017 23:14:27 +0100 Subject: [PATCH 059/230] bump version to 1.17 --- dotnet/Serpent/Properties/AssemblyInfo.cs | 4 ++-- dotnet/Serpent/Serpent.nuspec | 4 ++-- dotnet/Serpent/Version.cs | 2 +- java/src/main/java/net/razorvine/serpent/LibraryVersion.java | 2 +- java/src/main/java/net/razorvine/serpent/package-info.java | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dotnet/Serpent/Properties/AssemblyInfo.cs b/dotnet/Serpent/Properties/AssemblyInfo.cs index 1ffbf7e..6649bf4 100644 --- a/dotnet/Serpent/Properties/AssemblyInfo.cs +++ b/dotnet/Serpent/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ decoded there (using ast.literal_eval()). It can ofcourse also deserialize // // You can specify all the values or you can use the default the Revision and // Build Numbers by using the '*' as shown below: -[assembly: AssemblyVersion("1.16.0.*")] -[assembly: AssemblyFileVersion("1.16.0.0")] +[assembly: AssemblyVersion("1.17.0.*")] +[assembly: AssemblyFileVersion("1.17.0.0")] diff --git a/dotnet/Serpent/Serpent.nuspec b/dotnet/Serpent/Serpent.nuspec index 5c53595..673fab4 100644 --- a/dotnet/Serpent/Serpent.nuspec +++ b/dotnet/Serpent/Serpent.nuspec @@ -2,7 +2,7 @@ Razorvine.Serpent - 1.16.0.0 + 1.17.0.0 Razorvine.Serpent Irmen de Jong Irmen de Jong @@ -19,7 +19,7 @@ such a Python expression itself, back into the equivalent .NET datatypes. More info for the Python version is on Pypi: https://pypi.python.org/pypi/serpent Serpent Python literal expression serialization -Critical fix: fixed errors in complex number parser, it could crash on complex numbers containing float parts +@todo Copyright 2016 serialization python pyro diff --git a/dotnet/Serpent/Version.cs b/dotnet/Serpent/Version.cs index d0e61c4..2123caa 100644 --- a/dotnet/Serpent/Version.cs +++ b/dotnet/Serpent/Version.cs @@ -10,6 +10,6 @@ namespace Razorvine.Serpent { public static class LibraryVersion { - public static string Version = "1.16"; + public static string Version = "1.17"; } } diff --git a/java/src/main/java/net/razorvine/serpent/LibraryVersion.java b/java/src/main/java/net/razorvine/serpent/LibraryVersion.java index 71e5de8..a1b9a15 100644 --- a/java/src/main/java/net/razorvine/serpent/LibraryVersion.java +++ b/java/src/main/java/net/razorvine/serpent/LibraryVersion.java @@ -8,5 +8,5 @@ package net.razorvine.serpent; public final class LibraryVersion { - public static final String VERSION = "1.16"; + public static final String VERSION = "1.17"; } diff --git a/java/src/main/java/net/razorvine/serpent/package-info.java b/java/src/main/java/net/razorvine/serpent/package-info.java index 78e82bb..bd86c6e 100644 --- a/java/src/main/java/net/razorvine/serpent/package-info.java +++ b/java/src/main/java/net/razorvine/serpent/package-info.java @@ -3,7 +3,7 @@ * (a.k.a. Python's ast.literal_eval in Java) * Software license: "MIT software license". See http://opensource.org/licenses/MIT * @author Irmen de Jong (irmen@razorvine.net) - * @version 1.16 + * @version 1.17 */ package net.razorvine.serpent; From 82c0b0255cc9dbc10769b119aaa09a7e5ff01d78 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Mon, 13 Feb 2017 23:40:06 +0100 Subject: [PATCH 060/230] Added Serializer.MaximumLevel to avoid too deep recursion resulting in stack overflow errors. --- dotnet/Serpent.Test/CycleTest.cs | 37 ++++++++++++++++-- dotnet/Serpent/Serializer.cs | 22 +++++++++++ dotnet/Serpent/Serpent.nuspec | 2 +- .../net/razorvine/serpent/Serializer.java | 37 ++++++++++++++++++ .../net/razorvine/serpent/test/CycleTest.java | 38 +++++++++++++++++-- 5 files changed, 129 insertions(+), 7 deletions(-) diff --git a/dotnet/Serpent.Test/CycleTest.cs b/dotnet/Serpent.Test/CycleTest.cs index ff52bd6..fec62dd 100644 --- a/dotnet/Serpent.Test/CycleTest.cs +++ b/dotnet/Serpent.Test/CycleTest.cs @@ -60,7 +60,7 @@ public void testDictOk() } [Test] - [Ignore("stackoverflow")] + [ExpectedException(typeof(ArgumentException))] public void testListCycle() { var ser = new Serializer(); @@ -72,7 +72,7 @@ public void testListCycle() } [Test] - [Ignore("stackoverflow")] + [ExpectedException(typeof(ArgumentException))] public void testDictCycle() { var ser = new Serializer(); @@ -84,7 +84,7 @@ public void testDictCycle() } [Test] - [Ignore("stackoverflow")] + [ExpectedException(typeof(ArgumentException))] public void testClassCycle() { var ser = new Serializer(); @@ -95,5 +95,36 @@ public void testClassCycle() d.obj = d; var data = ser.Serialize(d); } + + [Test] + public void testMaxLevel() + { + Serializer ser = new Serializer(); + Assert.AreEqual(1000, ser.MaximumLevel); + + Object[] array = new Object[] { + "level1", + new Object[] { + "level2", + new Object[] { + "level3", + new Object[] { + "level 4" + } + } + } + }; + + ser.MaximumLevel = 4; + ser.Serialize(array); // should work + + ser.MaximumLevel = 3; + try { + ser.Serialize(array); + Assert.Fail("should fail"); + } catch(ArgumentException x) { + Assert.IsTrue(x.Message.Contains("too deep")); + } + } } } diff --git a/dotnet/Serpent/Serializer.cs b/dotnet/Serpent/Serializer.cs index 82617fe..6f1b165 100644 --- a/dotnet/Serpent/Serializer.cs +++ b/dotnet/Serpent/Serializer.cs @@ -24,9 +24,28 @@ namespace Razorvine.Serpent /// public class Serializer { + /// + /// indent output? + /// public bool Indent; + + /// + /// use set literals? + /// public bool SetLiterals; + + /// + /// include namespace prefix for classes that are serialized to dict? + /// public bool NamespaceInClassName; + + /// + /// The maximum nesting level of the object graphs that you want to serialize. + /// This limit has been set to avoid troublesome stack overflow errors. + /// (If it is reached, an IllegalArgumentException is thrown instead with a clear message) + /// + public int MaximumLevel = 1000; // avoids stackoverflow errors + private static IDictionary> classToDictRegistry = new Dictionary>(); @@ -73,6 +92,9 @@ public byte[] Serialize(object obj) protected void Serialize(object obj, TextWriter tw, int level) { + if(level>MaximumLevel) + throw new ArgumentException("Object graph nesting too deep. Increase serializer.MaximumLevel if you think you need more."); + // null -> None // hashtables/dictionaries -> dict // hashset -> set diff --git a/dotnet/Serpent/Serpent.nuspec b/dotnet/Serpent/Serpent.nuspec index 673fab4..07e1d20 100644 --- a/dotnet/Serpent/Serpent.nuspec +++ b/dotnet/Serpent/Serpent.nuspec @@ -19,7 +19,7 @@ such a Python expression itself, back into the equivalent .NET datatypes. More info for the Python version is on Pypi: https://pypi.python.org/pypi/serpent Serpent Python literal expression serialization -@todo +Added Serializer.MaximumLevel to avoid too deep recursion resulting in stack overflow errors. Copyright 2016 serialization python pyro diff --git a/java/src/main/java/net/razorvine/serpent/Serializer.java b/java/src/main/java/net/razorvine/serpent/Serializer.java index 21bc0b4..6c58c1d 100644 --- a/java/src/main/java/net/razorvine/serpent/Serializer.java +++ b/java/src/main/java/net/razorvine/serpent/Serializer.java @@ -30,15 +30,43 @@ */ public class Serializer { + /** + * The maximum nesting level of the object graphs that you want to serialize. + * This limit has been set to avoid troublesome stack overflow errors. + * (If it is reached, an IllegalArgumentException is thrown instead with a clear message) + */ + public int maximumLevel = 1000; // to avoid stack overflow errors + + /** + * Indent the resulting serpent serialization text? + */ public boolean indent = false; + + /** + * Use set literals? + */ public boolean setliterals = true; + + /** + * Include package name in class name, for classes that are serialized to dicts? + */ public boolean packageInClassName = false; + private static Map, IClassSerializer> classToDictRegistry = new HashMap, IClassSerializer>(); + /** + * Create a Serpent serializer with default options. + */ public Serializer() { } + /** + * Create a Serpent serializer with custom options. + * @param indent should the output be indented to make it more readable? + * @param setliterals should set literals be used (recommended if you use newer Python versions to parse this) + * @param packageInClassName should the package name be included with the class name for classes that are serialized to dict? + */ public Serializer(boolean indent, boolean setliterals, boolean packageInClassName) { this.indent = indent; @@ -46,11 +74,17 @@ public Serializer(boolean indent, boolean setliterals, boolean packageInClassNam this.packageInClassName = packageInClassName; } + /** + * Register a custom class serializer, if you want to tweak the serialization of classes that Serpent doesn't know about yet. + */ public static void registerClass(Class clazz, IClassSerializer converter) { classToDictRegistry.put(clazz, converter); } + /** + * Serialize an object graph to a serpent serialized form. + */ public byte[] serialize(Object obj) { StringWriter sw = new StringWriter(); @@ -77,6 +111,9 @@ public byte[] serialize(Object obj) protected void serialize(Object obj, PrintWriter p, int level) { + if(level>maximumLevel) + throw new IllegalArgumentException("Object graph nesting too deep. Increase serializer.maximumLevel if you think you need more."); + // null -> None // hashtables/dictionaries -> dict // hashset -> set diff --git a/java/src/test/java/net/razorvine/serpent/test/CycleTest.java b/java/src/test/java/net/razorvine/serpent/test/CycleTest.java index 317f9c6..d3f257d 100644 --- a/java/src/test/java/net/razorvine/serpent/test/CycleTest.java +++ b/java/src/test/java/net/razorvine/serpent/test/CycleTest.java @@ -13,6 +13,7 @@ import net.razorvine.serpent.Serializer; import org.junit.Test; +import static org.junit.Assert.*; public class CycleTest @@ -60,7 +61,7 @@ public void testDictOk() parser.parse(data); } - @Test(expected=StackOverflowError.class) + @Test(expected=IllegalArgumentException.class) public void testListCycle() { Serializer ser = new Serializer(); @@ -71,7 +72,7 @@ public void testListCycle() ser.serialize(d); } - @Test(expected=StackOverflowError.class) + @Test(expected=IllegalArgumentException.class) public void testDictCycle() { Serializer ser = new Serializer(); @@ -82,7 +83,7 @@ public void testDictCycle() ser.serialize(d); } - @Test(expected=StackOverflowError.class) + @Test(expected=IllegalArgumentException.class) public void testClassCycle() { Serializer ser = new Serializer(); @@ -93,4 +94,35 @@ public void testClassCycle() thing.obj = thing; ser.serialize(thing); } + + @Test + public void testMaxLevel() + { + Serializer ser = new Serializer(); + assertEquals(1000, ser.maximumLevel); + + Object[] array = new Object[] { + "level1", + new Object[] { + "level2", + new Object[] { + "level3", + new Object[] { + "level 4" + } + } + } + }; + + ser.maximumLevel = 4; + ser.serialize(array); // should work + + ser.maximumLevel = 3; + try { + ser.serialize(array); + fail("should fail"); + } catch(IllegalArgumentException x) { + assertTrue(x.getMessage().contains("too deep")); + } + } } From e4adb5a00ebb9f524f3d14774c4d5fc175cb8850 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 14 Feb 2017 02:56:05 +0100 Subject: [PATCH 061/230] add support for serializing some Jython classes that caused errors org.python.core.PyTuple org.python.core.PyComplex org.python.core.PyByteArray org.python.core.PyMemoryView --- .../net/razorvine/serpent/Serializer.java | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/java/src/main/java/net/razorvine/serpent/Serializer.java b/java/src/main/java/net/razorvine/serpent/Serializer.java index 6c58c1d..e3751b9 100644 --- a/java/src/main/java/net/razorvine/serpent/Serializer.java +++ b/java/src/main/java/net/razorvine/serpent/Serializer.java @@ -114,6 +114,9 @@ protected void serialize(Object obj, PrintWriter p, int level) if(level>maximumLevel) throw new IllegalArgumentException("Object graph nesting too deep. Increase serializer.maximumLevel if you think you need more."); + if(obj!=null && obj.getClass().getName().startsWith("org.python.")) + obj = convertJythonObject(obj); + // null -> None // hashtables/dictionaries -> dict // hashset -> set @@ -209,6 +212,51 @@ else if(obj instanceof Serializable) } } + /** + * When used from Jython directly, it sometimes passes some Jython specific + * classes to the serializer (such as org.python.core.PyComplex for a complex number). + * Due to the way these are constructed, Serpent is greatly confused, and will often + * end up in an endless loop eventually crashing with too deep nesting. + * For the know cases, we convert them here to appropriate representations. + */ + protected Object convertJythonObject(Object obj) + { + final Class clazz = obj.getClass(); + final String classname = clazz.getName(); + + try + { + // use reflection because I don't want to have a compiler dependency on Jython. + if(classname.equals("org.python.core.PyTuple")) { + return clazz.getMethod("toArray").invoke(obj); + } + else if(classname.equals("org.python.core.PyComplex")) { + Object pyImag = clazz.getMethod("getImag").invoke(obj); + Object pyReal = clazz.getMethod("getReal").invoke(obj); + Double imag = (Double) pyImag.getClass().getMethod("getValue").invoke(pyImag); + Double real = (Double) pyReal.getClass().getMethod("getValue").invoke(pyReal); + return new ComplexNumber(real, imag); + } + else if(classname.equals("org.python.core.PyByteArray")) { + Object pyStr = clazz.getMethod("__str__").invoke(obj); + return pyStr.getClass().getMethod("toBytes").invoke(pyStr); + } + else if(classname.equals("org.python.core.PyMemoryView")) { + Object pyBytes = clazz.getMethod("tobytes").invoke(obj); + return pyBytes.getClass().getMethod("toBytes").invoke(pyBytes); + } + } catch (ReflectiveOperationException e) { + throw new IllegalArgumentException("cannot serialize Jython object of type "+clazz, e); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("cannot serialize Jython object of type "+clazz, e); + } catch (SecurityException e) { + throw new IllegalArgumentException("cannot serialize Jython object of type "+clazz, e); + } + + // instead of an endless nesting loop, report a proper exception + throw new IllegalArgumentException("cannot serialize Jython object of type "+obj.getClass()); + } + protected void serialize_collection(Collection collection, PrintWriter p, int level) { // output a list From 9238b214289963e1b79eb6ed6944b15d0cb45653 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 14 Feb 2017 22:34:20 +0100 Subject: [PATCH 062/230] Raise a clear ValueError when the serpent string contains 0-bytes (due to corruption? bad serializer?) those cannot be parsed by ast.literal_eval. Also fixed a few ironpython tests. --- serpent.py | 4 +++- test_serpent.py | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/serpent.py b/serpent.py index 1540ed9..c0ef6c5 100644 --- a/serpent.py +++ b/serpent.py @@ -66,7 +66,7 @@ import array import math -__version__ = "1.16" +__version__ = "1.17" __all__ = ["dump", "dumps", "load", "loads", "register_class", "unregister_class", "tobytes"] can_use_set_literals = sys.version_info >= (3, 2) # check if we can use set literals @@ -85,6 +85,8 @@ def dump(obj, file, indent=False, set_literals=can_use_set_literals, module_in_c def loads(serialized_bytes): """Deserialize bytes back to object tree. Uses ast.literal_eval (safe).""" serialized = serialized_bytes.decode("utf-8") + if '\x00' in serialized: + raise ValueError("The serpent data contains 0-bytes so it cannot be parsed by ast.literal_eval. Has it been corrupted?") if sys.version_info < (3, 0) and sys.platform != "cli": if os.name == "java": # Because of a bug in Jython we have to manually convert all Str nodes to unicode. See http://bugs.jython.org/issue2008 diff --git a/test_serpent.py b/test_serpent.py index 167cccc..18425ac 100644 --- a/test_serpent.py +++ b/test_serpent.py @@ -199,6 +199,12 @@ def test_nullbytesunicode(self): data = serpent.loads(ser) self.assertEqual(line, data) + def test_detectNullByte(self): + with self.assertRaises(ValueError) as ex: + serpent.loads(b"contains\x00nullbyte") + self.fail("must fail") + self.assertTrue("0-bytes" in str(ex.exception)) + def test_unicode(self): u = "euro" + unichr(0x20ac) self.assertTrue(type(u) is unicode) @@ -257,7 +263,10 @@ def test_numbers(self): self.assertEqual(b"123456789123456789123456789", data) ser = serpent.dumps(99.1234) data = strip_header(ser) - self.assertEqual(b"99.1234", data) + if sys.platform == 'cli': + self.assertEqual(b"99.123400000000004", data) + else: + self.assertEqual(b"99.1234", data) ser = serpent.dumps(decimal.Decimal("1234.9999999999")) data = strip_header(ser) self.assertEqual(b"'1234.9999999999'", data) @@ -512,7 +521,10 @@ def test_time(self): self.assertEqual(b"'23:59:45'", data) ser = serpent.dumps(datetime.timedelta(1, 4000, 999888, minutes=22)) data = strip_header(ser) - self.assertEqual(b"91720.999888", data) + if sys.platform == 'cli': + self.assertEqual(b"91720.999888000006", data) + else: + self.assertEqual(b"91720.999888", data) ser = serpent.dumps(datetime.timedelta(seconds=12345)) data = strip_header(ser) self.assertEqual(b"12345.0", data) From 548c9191eec01c2b11de4021b30d7c02fb21a7e8 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Wed, 15 Feb 2017 01:50:12 +0100 Subject: [PATCH 063/230] Rewritten the way unicode strings are serialized. Now uses built-in repr() more often and properly escapes the 0x00 byte if it is in the text. (ast.literal_eval should no longer crash on it) --- serpent.py | 58 ++++++++++++++++++++++++++++++------------------- test_serpent.py | 28 ++++++++++++------------ test_unicode.py | 43 ++++++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 36 deletions(-) create mode 100644 test_unicode.py diff --git a/serpent.py b/serpent.py index c0ef6c5..36d1667 100644 --- a/serpent.py +++ b/serpent.py @@ -87,7 +87,7 @@ def loads(serialized_bytes): serialized = serialized_bytes.decode("utf-8") if '\x00' in serialized: raise ValueError("The serpent data contains 0-bytes so it cannot be parsed by ast.literal_eval. Has it been corrupted?") - if sys.version_info < (3, 0) and sys.platform != "cli": + if sys.version_info < (3, 0): if os.name == "java": # Because of a bug in Jython we have to manually convert all Str nodes to unicode. See http://bugs.jython.org/issue2008 serialized = ast.parse(serialized, "", mode="eval") @@ -353,27 +353,41 @@ def ser_builtins_complex(self, complex_obj, out, level): dispatch[complex] = ser_builtins_complex if sys.version_info < (3, 0): - def ser_builtins_unicode(self, unicode_obj, out, level): - # this method is used for python 2.x unicode (python 3.x doesn't use this) - z = unicode_obj.encode("utf-8") - # double-escape existing backslashes: - z = z.replace("\\", "\\\\") - # backslash-escape control characters: - z = z.replace("\a", "\\a") - z = z.replace("\b", "\\b") - z = z.replace("\f", "\\f") - z = z.replace("\n", "\\n") - z = z.replace("\r", "\\r") - z = z.replace("\t", "\\t") - z = z.replace("\v", "\\v") - if "'" not in z: - z = "'" + z + "'" - elif '"' not in z: - z = '"' + z + '"' - else: - z = z.replace("'", "\\'") - z = "'" + z + "'" - out.append(z) + # this method is used for python 2.x unicode (python 3.x doesn't use this) + if os.name == "java": + def ser_builtins_unicode(self, unicode_obj, out, level): + # because of a bug in Jython we cannot encode unicode literals. See http://bugs.jython.org/issue2008 + # workaround is to encode them first as utf-8 into a normal string, and when deserializing, + # decode all str nodes in the ast tree manually back into unicode. + z = unicode_obj.encode("utf-8") + # unfortunately we also cannot use repr() as it seems to cause errors later in the ast parser... + # that means we have to replicate the desired behavior here. + # double-escape existing backslashes: + z = z.replace("\\", "\\\\") + # backslash-escape control characters in the same way as repr() would: + z = z.replace("\a", "\\x07") + z = z.replace("\b", "\\x08") + z = z.replace("\f", "\\x0c") + z = z.replace("\n", "\\n") + z = z.replace("\r", "\\r") + z = z.replace("\t", "\\t") + z = z.replace("\v", "\\x0b") + # escape the 0-byte: + z = z.replace("\x00", "\\x00") + if "'" not in z: + z = "'" + z + "'" + elif '"' not in z: + z = '"' + z + '"' + else: + z = z.replace("'", "\\'") + z = "'" + z + "'" + out.append(z) + else: + def ser_builtins_unicode(self, unicode_obj, out, level): + z = repr(unicode_obj) + if z[0] == 'u': + z = z[1:] # get rid of the unicode 'u' prefix + out.append(z) dispatch[unicode] = ser_builtins_unicode if sys.version_info < (3, 0): diff --git a/test_serpent.py b/test_serpent.py index 18425ac..02e5ff4 100644 --- a/test_serpent.py +++ b/test_serpent.py @@ -170,32 +170,24 @@ def test_string_with_escapes(self): self.assertEqual(b"'\\n'", d) ser = serpent.dumps("\a") d = strip_header(ser) - if sys.platform == "cli": - self.assertEqual(b"'\\a'", d) # ironpython doesn't use repr() - else: - self.assertEqual(b"'\\x07'", d) # repr() does this hex escape + self.assertEqual(b"'\\x07'", d) # repr() does this hex escape line = "'hello\nlastline\ttab\\@slash\a\b\f\n\r\t\v'" ser = serpent.dumps(line) d = strip_header(ser) - if sys.platform == "cli": - self.assertEqual(b"\"'hello\\nlastline\\ttab\\\\@slash\\a\\b\\f\\n\\r\\t\\v'\"", d) # ironpython doesn't use repr() - else: - self.assertEqual(b"\"'hello\\nlastline\\ttab\\\\@slash\\x07\\x08\\x0c\\n\\r\\t\\x0b'\"", d) # the hex escapes are done by repr() + self.assertEqual(b"\"'hello\\nlastline\\ttab\\\\@slash\\x07\\x08\\x0c\\n\\r\\t\\x0b'\"", d) # the hex escapes are done by repr() data = serpent.loads(ser) self.assertEqual(line, data) - @unittest.skipIf(sys.platform == "cli", "IronPython has problems with null bytes in strings") def test_nullbytesstring(self): ser = serpent.dumps("\0null") data = serpent.loads(ser) self.assertEqual("\0null", data) - @unittest.skipIf(sys.version_info < (3, 0), "needs python 3.x to correctly process null bytes in unicode strings") def test_nullbytesunicode(self): line = unichr(0) + "null" ser = serpent.dumps(line) data = strip_header(ser) - self.assertEqual(b"'\\x00null'", data) + self.assertEqual(b"'\\x00null'", data, "must escape 0-byte") data = serpent.loads(ser) self.assertEqual(line, data) @@ -225,14 +217,22 @@ def test_unicode_with_escapes_py2(self): self.assertEqual(b"'\\n'", d) ser = serpent.dumps(unicode("\a")) d = strip_header(ser) - self.assertEqual(b"'\\a'", d) + self.assertEqual(b"'\\x07'", d) ser = serpent.dumps("\a"+unichr(0x20ac)) d = strip_header(ser) - self.assertEqual(b"'\\a\xe2\x82\xac'", d) + if os.name == "java": + # different because of Jython bug workaround + self.assertEqual(b"'\\x07\xe2\x82\xac'", d) + else: + self.assertEqual(b"'\\x07\\u20ac'", d) line = "'euro" + unichr(0x20ac) + "\nlastline\ttab\\@slash\a\b\f\n\r\t\v'" ser = serpent.dumps(line) d = strip_header(ser) - self.assertEqual(b"\"'euro\xe2\x82\xac\\nlastline\\ttab\\\\@slash\\a\\b\\f\\n\\r\\t\\v'\"", d) + if os.name=="java": + # different because of Jython bug workaround + self.assertEqual(b"\"'euro\xe2\x82\xac\\nlastline\\ttab\\\\@slash\\x07\\x08\\x0c\\n\\r\\t\\x0b'\"", d) + else: + self.assertEqual(b"\"'euro\\u20ac\\nlastline\\ttab\\\\@slash\\x07\\x08\\x0c\\n\\r\\t\\x0b'\"", d) data = serpent.loads(ser) self.assertEqual(line, data) diff --git a/test_unicode.py b/test_unicode.py new file mode 100644 index 0000000..0defd74 --- /dev/null +++ b/test_unicode.py @@ -0,0 +1,43 @@ +from __future__ import print_function +import sys +import serpent +import platform + +impl=platform.python_implementation()+"_{0}_{1}".format(sys.version_info[0], sys.version_info[1]) +print("IMPL:", impl) + +if sys.version_info>=(3,0): + unichr = chr + +teststrings = [ + u"", + u"abc", + u"\u20ac", + u"\x00\x01\x80\x81\xfe\xff" +] + +large = u"".join(unichr(i) for i in range(256)) +teststrings.append(large) +large = u"".join(unichr(i) for i in range(0x20ac+1)) +teststrings.append(large) + +with open("data_inputs_utf8.txt", "wb") as out: + for source in teststrings: + out.write(source.encode("utf-8")+b"\n") + +results = [] +ser = serpent.Serializer() +with open("data_"+impl+".serpent", "wb") as out: + for i, source in enumerate(teststrings): + data = ser.serialize(source) + out.write(data) + out.write(b"\n\n") + assert b"\x00" not in data + results.append(data) + +assert len(results)==len(teststrings) +for i, source in enumerate(teststrings): + print(i) + result = serpent.loads(results[i]) + assert type(source) is type(result) + assert source==result From 5359cb7afa4279a79150a5414ae0ced0ef845b39 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Wed, 15 Feb 2017 01:58:57 +0100 Subject: [PATCH 064/230] moved tests files to new subdir --- Makefile | 4 +- README.md | 2 +- example.py => tests/example.py | 76 +++++++++---------- performance.py => tests/performance.py | 0 test_serpent.py => tests/test_serpent.py | 0 test_unicode.py => tests/test_unicode.py | 0 .../testserpent.utf8.bin | 0 tox.ini | 4 +- 8 files changed, 43 insertions(+), 43 deletions(-) rename example.py => tests/example.py (96%) rename performance.py => tests/performance.py (100%) rename test_serpent.py => tests/test_serpent.py (100%) rename test_unicode.py => tests/test_unicode.py (100%) rename testserpent.utf8.bin => tests/testserpent.utf8.bin (100%) diff --git a/Makefile b/Makefile index 1ef4e70..f84da02 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: all sdist wheel install upload clean test check all: - @echo "targets include sdist, wheel, upload, install, clean" + @echo "targets include sdist, wheel, upload, install, clean, test" sdist: python setup.py sdist @@ -19,7 +19,7 @@ install: python setup.py install test: - python -E test_serpent.py + cd tests && python -E test_serpent.py check: flake8 --exclude .tox --ignore E501 diff --git a/README.md b/README.md index ee1e72f..e0a052f 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ This license, including disclaimer, is available in the 'LICENSE' file. PYTHON ------ Package can be found on Pypi as 'serpent': https://pypi.python.org/pypi/serpent -Example usage can be found in ./example.py +Example usage can be found in ./tests/example.py C#/.NET diff --git a/example.py b/tests/example.py similarity index 96% rename from example.py rename to tests/example.py index 01b290f..40ca207 100644 --- a/example.py +++ b/tests/example.py @@ -1,38 +1,38 @@ -from __future__ import print_function -import datetime -import serpent - - -class CustomClass(object): - def __init__(self, name, age): - self.name = name - self.age = age - - -def example(): - data = { - "tuple": (1, 2, 3), - "date": datetime.datetime.now(), - "set": set(['a', 'b', 'c']), - "class": CustomClass("Sally", 26) - } - - # serialize the object - ser = serpent.dumps(data, indent=True) - # print it to the screen, but usually you'd save the bytes to a file or transfer them over a network connection - print("Serialized data:") - print(ser.decode("UTF-8")) - - # deserialize the bytes and print the objects - obj = serpent.loads(ser) - print("Deserialized data:") - print("tuple:", obj["tuple"]) - print("date:", obj["date"]) - print("set:", obj["set"]) - clazz = obj["class"] - print("class attributes: type={0} name={1} age={2}".format( - clazz["__class__"], clazz["name"], clazz["age"])) - - -if __name__ == "__main__": - example() +from __future__ import print_function +import datetime +import serpent + + +class CustomClass(object): + def __init__(self, name, age): + self.name = name + self.age = age + + +def example(): + data = { + "tuple": (1, 2, 3), + "date": datetime.datetime.now(), + "set": set(['a', 'b', 'c']), + "class": CustomClass("Sally", 26) + } + + # serialize the object + ser = serpent.dumps(data, indent=True) + # print it to the screen, but usually you'd save the bytes to a file or transfer them over a network connection + print("Serialized data:") + print(ser.decode("UTF-8")) + + # deserialize the bytes and print the objects + obj = serpent.loads(ser) + print("Deserialized data:") + print("tuple:", obj["tuple"]) + print("date:", obj["date"]) + print("set:", obj["set"]) + clazz = obj["class"] + print("class attributes: type={0} name={1} age={2}".format( + clazz["__class__"], clazz["name"], clazz["age"])) + + +if __name__ == "__main__": + example() diff --git a/performance.py b/tests/performance.py similarity index 100% rename from performance.py rename to tests/performance.py diff --git a/test_serpent.py b/tests/test_serpent.py similarity index 100% rename from test_serpent.py rename to tests/test_serpent.py diff --git a/test_unicode.py b/tests/test_unicode.py similarity index 100% rename from test_unicode.py rename to tests/test_unicode.py diff --git a/testserpent.utf8.bin b/tests/testserpent.utf8.bin similarity index 100% rename from testserpent.utf8.bin rename to tests/testserpent.utf8.bin diff --git a/tox.ini b/tox.ini index e3bc339..a4eb532 100644 --- a/tox.ini +++ b/tox.ini @@ -2,10 +2,10 @@ envlist=py27,py33,py34,py35,py36,pypy [testenv] -commands=python -bb -E test_serpent.py deps=pytz +changedir=tests +commands=python -bb -E test_serpent.py [testenv:py26] deps=unittest2 pytz - From 4ed5f3a5db11c9830d620ac60027cd6c829b6d27 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Wed, 15 Feb 2017 02:04:17 +0100 Subject: [PATCH 065/230] tox fix? --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index a4eb532..5f41c91 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ envlist=py27,py33,py34,py35,py36,pypy [testenv] deps=pytz -changedir=tests +changedir={toxinidir}/tests commands=python -bb -E test_serpent.py [testenv:py26] From a0f82df5e76e7a607570dbf75f7e98db749ea5b3 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Wed, 15 Feb 2017 02:08:16 +0100 Subject: [PATCH 066/230] travis fix? --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ba7ec44..d9b8721 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,4 +15,4 @@ install: - pip install pytz - pip install . -script: python -bb test_serpent.py +script: cd tests && python -bb test_serpent.py From a96210a79f7cbdde83f540eedc83e0707c649994 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Wed, 15 Feb 2017 02:08:48 +0100 Subject: [PATCH 067/230] ? --- tests/example.py | 76 ++++++++++++++++++++++++------------------------ 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/tests/example.py b/tests/example.py index 40ca207..01b290f 100644 --- a/tests/example.py +++ b/tests/example.py @@ -1,38 +1,38 @@ -from __future__ import print_function -import datetime -import serpent - - -class CustomClass(object): - def __init__(self, name, age): - self.name = name - self.age = age - - -def example(): - data = { - "tuple": (1, 2, 3), - "date": datetime.datetime.now(), - "set": set(['a', 'b', 'c']), - "class": CustomClass("Sally", 26) - } - - # serialize the object - ser = serpent.dumps(data, indent=True) - # print it to the screen, but usually you'd save the bytes to a file or transfer them over a network connection - print("Serialized data:") - print(ser.decode("UTF-8")) - - # deserialize the bytes and print the objects - obj = serpent.loads(ser) - print("Deserialized data:") - print("tuple:", obj["tuple"]) - print("date:", obj["date"]) - print("set:", obj["set"]) - clazz = obj["class"] - print("class attributes: type={0} name={1} age={2}".format( - clazz["__class__"], clazz["name"], clazz["age"])) - - -if __name__ == "__main__": - example() +from __future__ import print_function +import datetime +import serpent + + +class CustomClass(object): + def __init__(self, name, age): + self.name = name + self.age = age + + +def example(): + data = { + "tuple": (1, 2, 3), + "date": datetime.datetime.now(), + "set": set(['a', 'b', 'c']), + "class": CustomClass("Sally", 26) + } + + # serialize the object + ser = serpent.dumps(data, indent=True) + # print it to the screen, but usually you'd save the bytes to a file or transfer them over a network connection + print("Serialized data:") + print(ser.decode("UTF-8")) + + # deserialize the bytes and print the objects + obj = serpent.loads(ser) + print("Deserialized data:") + print("tuple:", obj["tuple"]) + print("date:", obj["date"]) + print("set:", obj["set"]) + clazz = obj["class"] + print("class attributes: type={0} name={1} age={2}".format( + clazz["__class__"], clazz["name"], clazz["age"])) + + +if __name__ == "__main__": + example() From 992be2d51d03d2eff2104d8661ceb80dade65dde Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Wed, 15 Feb 2017 23:08:00 +0100 Subject: [PATCH 068/230] jython unicode ser changes --- serpent.py | 47 ++++++++++++++++++++++-------------- tests/test_serpent.py | 30 +++++++++++++++++++++-- tests/test_unicode.py | 48 +++++++++++++++++++++---------------- tests/test_unicode_parse.py | 25 +++++++++++++++++++ 4 files changed, 109 insertions(+), 41 deletions(-) create mode 100644 tests/test_unicode_parse.py diff --git a/serpent.py b/serpent.py index 36d1667..e2507ac 100644 --- a/serpent.py +++ b/serpent.py @@ -363,25 +363,36 @@ def ser_builtins_unicode(self, unicode_obj, out, level): # unfortunately we also cannot use repr() as it seems to cause errors later in the ast parser... # that means we have to replicate the desired behavior here. # double-escape existing backslashes: - z = z.replace("\\", "\\\\") - # backslash-escape control characters in the same way as repr() would: - z = z.replace("\a", "\\x07") - z = z.replace("\b", "\\x08") - z = z.replace("\f", "\\x0c") - z = z.replace("\n", "\\n") - z = z.replace("\r", "\\r") - z = z.replace("\t", "\\t") - z = z.replace("\v", "\\x0b") - # escape the 0-byte: - z = z.replace("\x00", "\\x00") - if "'" not in z: - z = "'" + z + "'" - elif '"' not in z: - z = '"' + z + '"' + b = [] + contains_single_quote = False + contains_quote = False + for c in z: + if c == '\\': + b.append("\\\\") + elif c == '\n': + b.append("\\n") + elif c == '\r': + b.append("\\r") + elif c == '\t': + b.append("\\t") + elif c < ' ': + b.append("\\x%02x" % ord(c)) + else: + b.append(c) + contains_single_quote |= c == '\'' + contains_quote |= c == '"' + if not contains_single_quote: + out.append('\'') + out.append("".join(b)) + out.append('\'') + elif not contains_quote: + out.append('"') + out.append("".join(b)) + out.append('"') else: - z = z.replace("'", "\\'") - z = "'" + z + "'" - out.append(z) + out.append('\'') + out.append("".join(b).replace("\'", "\\'")) + out.append('\'') else: def ser_builtins_unicode(self, unicode_obj, out, level): z = repr(unicode_obj) diff --git a/tests/test_serpent.py b/tests/test_serpent.py index 02e5ff4..757704a 100644 --- a/tests/test_serpent.py +++ b/tests/test_serpent.py @@ -6,6 +6,7 @@ """ from __future__ import print_function, division import sys +import ast import timeit import datetime import uuid @@ -179,9 +180,21 @@ def test_string_with_escapes(self): self.assertEqual(line, data) def test_nullbytesstring(self): - ser = serpent.dumps("\0null") + ser = serpent.dumps(u"\x00null") data = serpent.loads(ser) - self.assertEqual("\0null", data) + self.assertEqual("\x00null", data) + ser = serpent.dumps(u"\x01") + self.assertEqual(b"'\\x01'", strip_header(ser)) + data = serpent.loads(ser) + self.assertEqual("\x01", data) + ser = serpent.dumps(u"\x1f") + self.assertEqual(b"'\\x1f'", strip_header(ser)) + data = serpent.loads(ser) + self.assertEqual("\x1f", data) + ser = serpent.dumps(u"\x20") + self.assertEqual(b"' '", strip_header(ser)) + data = serpent.loads(ser) + self.assertEqual(" ", data) def test_nullbytesunicode(self): line = unichr(0) + "null" @@ -210,6 +223,19 @@ def test_unicode(self): data = strip_header(ser) self.assertEqual(b"\"quotes2'\"", data) + def test_utf8_correctness(self): + u = u"\x00\x01\x80\x81\xfe\xffabcdef\u20ac" + utf_8_correct = repr(u).encode("utf-8") + if utf_8_correct.startswith(b"u"): + utf_8_correct=utf_8_correct[1:] + ser = serpent.dumps(u) + d = strip_header(ser) + print("\nSERPENT:", d) # XXX + print("\nCORRECT=>", repr(utf_8_correct)) + print(" D=>", repr(d)) + self.assertEqual(utf_8_correct, d) + + @unittest.skipIf(sys.version_info >= (3, 0), "py2 escaping tested") def test_unicode_with_escapes_py2(self): ser = serpent.dumps(unicode("\n")) diff --git a/tests/test_unicode.py b/tests/test_unicode.py index 0defd74..443aca3 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -3,9 +3,6 @@ import serpent import platform -impl=platform.python_implementation()+"_{0}_{1}".format(sys.version_info[0], sys.version_info[1]) -print("IMPL:", impl) - if sys.version_info>=(3,0): unichr = chr @@ -21,23 +18,32 @@ large = u"".join(unichr(i) for i in range(0x20ac+1)) teststrings.append(large) -with open("data_inputs_utf8.txt", "wb") as out: - for source in teststrings: - out.write(source.encode("utf-8")+b"\n") -results = [] -ser = serpent.Serializer() -with open("data_"+impl+".serpent", "wb") as out: +def main(): + impl=platform.python_implementation()+"_{0}_{1}".format(sys.version_info[0], sys.version_info[1]) + print("IMPL:", impl) + + with open("data_inputs_utf8.txt", "wb") as out: + for source in teststrings: + out.write(source.encode("utf-8")+b"\n") + + results = [] + ser = serpent.Serializer() + with open("data_"+impl+".serpent", "wb") as out: + for i, source in enumerate(teststrings): + data = ser.serialize(source) + out.write(data) + out.write(b"\n\n") + assert b"\x00" not in data + results.append(data) + + assert len(results)==len(teststrings) for i, source in enumerate(teststrings): - data = ser.serialize(source) - out.write(data) - out.write(b"\n\n") - assert b"\x00" not in data - results.append(data) - -assert len(results)==len(teststrings) -for i, source in enumerate(teststrings): - print(i) - result = serpent.loads(results[i]) - assert type(source) is type(result) - assert source==result + print(i) + result = serpent.loads(results[i]) + assert type(source) is type(result) + assert source==result + print("OK") + +if __name__ == "__main__": + main() diff --git a/tests/test_unicode_parse.py b/tests/test_unicode_parse.py new file mode 100644 index 0000000..a1a45e6 --- /dev/null +++ b/tests/test_unicode_parse.py @@ -0,0 +1,25 @@ +import os +import io +import serpent +from test_unicode import teststrings + + +files = [f for f in os.listdir(".") if f.startswith("data_") and f.endswith(".serpent")] + +for f in files: + print("Checking data file", f) + resultstrings=[] + with io.open(f, "rb") as inf: + data = inf.read() + data = data.split(b"\n\n")[:-1] + # data = data[:-2] # XXX + for d in data: + try: + resultstrings.append(serpent.loads(d)) + except Exception as x: + print("SERPENT ERROR", type(x)) + + if resultstrings==teststrings: + print("OK") + else: + print("!!!FAIL!!!") From f8c2c0bd2a20a88bab536b2d30a27e8f468182b1 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Thu, 16 Feb 2017 01:39:37 +0100 Subject: [PATCH 069/230] Reworked the serialization of unicode strings. Fixed utf-8 related encoding bugs. For Jython however now has to raise ValueError when a unicode string is deserialized because Jython's ast module doesn't work correctly. --- serpent.py | 77 ++++++++++++------------------------- tests/test_serpent.py | 35 +++++++++-------- tests/test_unicode.py | 11 ++++-- tests/test_unicode_parse.py | 10 +++-- 4 files changed, 59 insertions(+), 74 deletions(-) diff --git a/serpent.py b/serpent.py index e2507ac..8da6205 100644 --- a/serpent.py +++ b/serpent.py @@ -84,16 +84,29 @@ def dump(obj, file, indent=False, set_literals=can_use_set_literals, module_in_c def loads(serialized_bytes): """Deserialize bytes back to object tree. Uses ast.literal_eval (safe).""" - serialized = serialized_bytes.decode("utf-8") - if '\x00' in serialized: + if b'\x00' in serialized_bytes: raise ValueError("The serpent data contains 0-bytes so it cannot be parsed by ast.literal_eval. Has it been corrupted?") + serialized = serialized_bytes.decode("utf-8") if sys.version_info < (3, 0): if os.name == "java": - # Because of a bug in Jython we have to manually convert all Str nodes to unicode. See http://bugs.jython.org/issue2008 - serialized = ast.parse(serialized, "", mode="eval") - for node in ast.walk(serialized): - if isinstance(node, ast.Str) and type(node.s) is str: - node.s = node.s.decode("utf-8") + # Because of a bug in Jython we cannot correctly parse strings with characters >255 in them. + # These are encoded using the \u???? escape sequence, and should be parsed back into a proper + # unicode string. However the ast module in Jython is unable to do this correctly. + # See http://bugs.jython.org/issue2008 + # I tried various things (unicode-escaping, re-encoding in utf-8, modifying the ast syntax tree to + # try to convert the str nodes, but all seems in vain. The parsing simply processes the unicode wrong, + # and loses information in the process that cannot be restored by postprocessing. + # The only option for now seems to be to give up if we encounter such a string. + # So for Jython: only use ASCII strings or encode/decode the unicode strings yourself into bytes or ASCII + # before sending them through Serpent. + import re + if re.search(r"\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}", serialized): + raise ValueError("Unsupported serialized string encountered. Unfortunately it is currently " \ + "impossible to deserialize a string with characters above value 255 in it, " \ + "due to a bug with the ast module in Jython. Either change the string that is " \ + "given to this application (for instance make sure it is ASCII only), " \ + "or wait until the bug in Jython is fixed: http://bugs.jython.org/issue2008 "\ + "(Unfortunately there seems little progress in solving the bug)") else: # python 2.x: parse with unicode_literals (promotes all strings to unicode) serialized = compile(serialized, "", mode="eval", flags=ast.PyCF_ONLY_AST | __future__.unicode_literals.compiler_flag) @@ -354,51 +367,11 @@ def ser_builtins_complex(self, complex_obj, out, level): if sys.version_info < (3, 0): # this method is used for python 2.x unicode (python 3.x doesn't use this) - if os.name == "java": - def ser_builtins_unicode(self, unicode_obj, out, level): - # because of a bug in Jython we cannot encode unicode literals. See http://bugs.jython.org/issue2008 - # workaround is to encode them first as utf-8 into a normal string, and when deserializing, - # decode all str nodes in the ast tree manually back into unicode. - z = unicode_obj.encode("utf-8") - # unfortunately we also cannot use repr() as it seems to cause errors later in the ast parser... - # that means we have to replicate the desired behavior here. - # double-escape existing backslashes: - b = [] - contains_single_quote = False - contains_quote = False - for c in z: - if c == '\\': - b.append("\\\\") - elif c == '\n': - b.append("\\n") - elif c == '\r': - b.append("\\r") - elif c == '\t': - b.append("\\t") - elif c < ' ': - b.append("\\x%02x" % ord(c)) - else: - b.append(c) - contains_single_quote |= c == '\'' - contains_quote |= c == '"' - if not contains_single_quote: - out.append('\'') - out.append("".join(b)) - out.append('\'') - elif not contains_quote: - out.append('"') - out.append("".join(b)) - out.append('"') - else: - out.append('\'') - out.append("".join(b).replace("\'", "\\'")) - out.append('\'') - else: - def ser_builtins_unicode(self, unicode_obj, out, level): - z = repr(unicode_obj) - if z[0] == 'u': - z = z[1:] # get rid of the unicode 'u' prefix - out.append(z) + def ser_builtins_unicode(self, unicode_obj, out, level): + z = repr(unicode_obj) + if z[0] == 'u': + z = z[1:] # get rid of the unicode 'u' prefix + out.append(z) dispatch[unicode] = ser_builtins_unicode if sys.version_info < (3, 0): diff --git a/tests/test_serpent.py b/tests/test_serpent.py index 757704a..ed6e35f 100644 --- a/tests/test_serpent.py +++ b/tests/test_serpent.py @@ -46,6 +46,9 @@ class TestDeserialize(unittest.TestCase): def test_deserialize(self): data = serpent.loads(b"555") self.assertEqual(555, data) + + @unittest.skipIf(os.name=="java", "Jython can't deserialize strings with unicode in them, bug #2008") + def test_deserialize_unichr(self): unicodestring = "euro" + unichr(0x20ac) encoded = repr(unicodestring).encode("utf-8") data = serpent.loads(encoded) @@ -73,6 +76,13 @@ def test_trailing_comma_set(self): v = serpent.loads(b"{1,2,3,}") self.assertEqual(set([1, 2, 3]), v) + @unittest.skipIf(os.name=="java", "Jython can't deserialize strings with unicode in them, bug #2008") + def test_unicode_escapes(self): + v = serpent.loads(b"'\u20ac'") + self.assertEqual(u"\u20ac", v) + v = serpent.loads(b"'\U00022001'") + self.assertEqual(u"\U00022001", v) + class TestBasics(unittest.TestCase): def test_header(self): @@ -210,12 +220,15 @@ def test_detectNullByte(self): self.fail("must fail") self.assertTrue("0-bytes" in str(ex.exception)) + @unittest.skipIf(os.name=="java", "Jython can't deserialize strings with unicode in them, bug #2008") def test_unicode(self): - u = "euro" + unichr(0x20ac) + u = "euro" + unichr(0x20ac)+"\U00022001" self.assertTrue(type(u) is unicode) ser = serpent.dumps(u) data = serpent.loads(ser) self.assertEqual(u, data) + + def test_unicode_quotes(self): ser = serpent.dumps(unicode("quotes'\"")) data = strip_header(ser) self.assertEqual(b"'quotes\\'\"'", data) @@ -230,12 +243,8 @@ def test_utf8_correctness(self): utf_8_correct=utf_8_correct[1:] ser = serpent.dumps(u) d = strip_header(ser) - print("\nSERPENT:", d) # XXX - print("\nCORRECT=>", repr(utf_8_correct)) - print(" D=>", repr(d)) self.assertEqual(utf_8_correct, d) - @unittest.skipIf(sys.version_info >= (3, 0), "py2 escaping tested") def test_unicode_with_escapes_py2(self): ser = serpent.dumps(unicode("\n")) @@ -244,21 +253,17 @@ def test_unicode_with_escapes_py2(self): ser = serpent.dumps(unicode("\a")) d = strip_header(ser) self.assertEqual(b"'\\x07'", d) + + @unittest.skipIf(os.name=="java", "Jython can't deserialize strings with unicode in them, bug #2008") + @unittest.skipIf(sys.version_info >= (3, 0), "py2 escaping tested") + def test_unicode_with_escapes_unichrs(self): ser = serpent.dumps("\a"+unichr(0x20ac)) d = strip_header(ser) - if os.name == "java": - # different because of Jython bug workaround - self.assertEqual(b"'\\x07\xe2\x82\xac'", d) - else: - self.assertEqual(b"'\\x07\\u20ac'", d) + self.assertEqual(b"'\\x07\\u20ac'", d) line = "'euro" + unichr(0x20ac) + "\nlastline\ttab\\@slash\a\b\f\n\r\t\v'" ser = serpent.dumps(line) d = strip_header(ser) - if os.name=="java": - # different because of Jython bug workaround - self.assertEqual(b"\"'euro\xe2\x82\xac\\nlastline\\ttab\\\\@slash\\x07\\x08\\x0c\\n\\r\\t\\x0b'\"", d) - else: - self.assertEqual(b"\"'euro\\u20ac\\nlastline\\ttab\\\\@slash\\x07\\x08\\x0c\\n\\r\\t\\x0b'\"", d) + self.assertEqual(b"\"'euro\\u20ac\\nlastline\\ttab\\\\@slash\\x07\\x08\\x0c\\n\\r\\t\\x0b'\"", d) data = serpent.loads(ser) self.assertEqual(line, data) diff --git a/tests/test_unicode.py b/tests/test_unicode.py index 443aca3..55219b1 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -10,7 +10,7 @@ u"", u"abc", u"\u20ac", - u"\x00\x01\x80\x81\xfe\xff" + u"\x00\x01\x80\x81\xfe\xff\u20ac\u4444\u0240slashu:\\uend.\\u20ac(no euro!)\\U00022001bigone" ] large = u"".join(unichr(i) for i in range(256)) @@ -33,7 +33,7 @@ def main(): for i, source in enumerate(teststrings): data = ser.serialize(source) out.write(data) - out.write(b"\n\n") + out.write(b"~\n~\n") assert b"\x00" not in data results.append(data) @@ -41,8 +41,11 @@ def main(): for i, source in enumerate(teststrings): print(i) result = serpent.loads(results[i]) - assert type(source) is type(result) - assert source==result + if source!=result: + print("ERRROR!!! RESULT AFTER serpent.loads IS NOT CORRECT!") + print("SOURCE:",repr(source)) + print("RESULT:",repr(result)) + return print("OK") if __name__ == "__main__": diff --git a/tests/test_unicode_parse.py b/tests/test_unicode_parse.py index a1a45e6..11e5fd5 100644 --- a/tests/test_unicode_parse.py +++ b/tests/test_unicode_parse.py @@ -1,5 +1,7 @@ +from __future__ import print_function import os import io +import re import serpent from test_unicode import teststrings @@ -11,13 +13,15 @@ resultstrings=[] with io.open(f, "rb") as inf: data = inf.read() - data = data.split(b"\n\n")[:-1] + data = re.split(b"~\n~\n", data)[:-1] + assert len(data) == len(teststrings) # data = data[:-2] # XXX - for d in data: + for num, d in enumerate(data, start=1): try: + print("data item ",num,"...") resultstrings.append(serpent.loads(d)) except Exception as x: - print("SERPENT ERROR", type(x)) + print("\nSERPENT ERROR", type(x)) if resultstrings==teststrings: print("OK") From e47e91a12b1011406b37cb385577d368860cb85d Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Thu, 16 Feb 2017 22:31:06 +0100 Subject: [PATCH 070/230] java: reworked string repr --- .../net/razorvine/serpent/Serializer.java | 86 ++++++++++++----- .../razorvine/serpent/test/SerializeTest.java | 95 +++++++++++++++---- 2 files changed, 140 insertions(+), 41 deletions(-) diff --git a/java/src/main/java/net/razorvine/serpent/Serializer.java b/java/src/main/java/net/razorvine/serpent/Serializer.java index e3751b9..431d178 100644 --- a/java/src/main/java/net/razorvine/serpent/Serializer.java +++ b/java/src/main/java/net/razorvine/serpent/Serializer.java @@ -90,16 +90,14 @@ public byte[] serialize(Object obj) StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); - String header = "# serpent utf-8 "; if(this.setliterals) - header += "python3.2\n"; // set-literals require python 3.2+ to deserialize (ast.literal_eval limitation) + pw.print("# serpent utf-8 python3.2\n"); // set-literals require python 3.2+ to deserialize (ast.literal_eval limitation) else - header += "python2.6\n"; - pw.print(header); + pw.print("# serpent utf-8 python2.6\n"); serialize(obj, pw, 0); pw.flush(); - String ser = sw.toString(); + final String ser = sw.toString(); pw.close(); try { sw.close(); @@ -585,26 +583,70 @@ else if(d.isNaN()) { p.print(obj); } } + + // the repr translation table for characters 0x00-0xff + private final static String[] repr_255; + static { + repr_255=new String[256]; + for(int c=0; c<32; ++c) { + repr_255[c] = String.format("\\x%02x",c); + } + for(char c=0x20; c<0x7f; ++c) { + repr_255[c] = String.valueOf(c); + } + for(int c=0x7f; c<=0xa0; ++c) { + repr_255[c] = String.format("\\x%02x", c); + } + for(char c=0xa1; c<=0xff; ++c) { + repr_255[c] = String.valueOf(c); + } + // odd ones out: + repr_255['\t'] = "\\t"; + repr_255['\n'] = "\\n"; + repr_255['\r'] = "\\r"; + repr_255['\\'] = "\\\\"; + repr_255[0xad] = "\\0xad"; + } protected void serialize_string(String str, PrintWriter p, int level) { - // backslash-escaped string - str = str.replace("\\", "\\\\"); // double-escape the backslashes - str = str.replace("\b", "\\b"); - str = str.replace("\f", "\\f"); - str = str.replace("\n", "\\n"); - str = str.replace("\r", "\\r"); - str = str.replace("\t", "\\t"); - if(!str.contains("'")) - str = "'" + str + "'"; - else if(!str.contains("\"")) - str = '"' + str + '"'; - else - { - str = str.replace("'", "\\'"); - str = "'" + str + "'"; - } - p.print(str); + // create a 'repr' string representation following the same escaping rules as python 3.x repr() does. + StringBuilder b=new StringBuilder(str.length()); + boolean containsSingleQuote=false; + boolean containsQuote=false; + for(int i=0; i Date: Thu, 16 Feb 2017 22:45:20 +0100 Subject: [PATCH 071/230] java: optimized serializer io a little bit --- .../net/razorvine/serpent/Serializer.java | 209 +++++++++--------- .../net/razorvine/serpent/test/CycleTest.java | 2 +- .../razorvine/serpent/test/SerializeTest.java | 2 +- 3 files changed, 105 insertions(+), 108 deletions(-) diff --git a/java/src/main/java/net/razorvine/serpent/Serializer.java b/java/src/main/java/net/razorvine/serpent/Serializer.java index 431d178..d81186f 100644 --- a/java/src/main/java/net/razorvine/serpent/Serializer.java +++ b/java/src/main/java/net/razorvine/serpent/Serializer.java @@ -8,7 +8,6 @@ package net.razorvine.serpent; import java.io.IOException; -import java.io.PrintWriter; import java.io.Serializable; import java.io.StringWriter; import java.lang.reflect.Array; @@ -35,7 +34,7 @@ public class Serializer * This limit has been set to avoid troublesome stack overflow errors. * (If it is reached, an IllegalArgumentException is thrown instead with a clear message) */ - public int maximumLevel = 1000; // to avoid stack overflow errors + public int maximumLevel = 800; // to avoid stack overflow errors /** * Indent the resulting serpent serialization text? @@ -88,17 +87,15 @@ public static void registerClass(Class clazz, IClassSerializer converter) public byte[] serialize(Object obj) { StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); if(this.setliterals) - pw.print("# serpent utf-8 python3.2\n"); // set-literals require python 3.2+ to deserialize (ast.literal_eval limitation) + sw.write("# serpent utf-8 python3.2\n"); // set-literals require python 3.2+ to deserialize (ast.literal_eval limitation) else - pw.print("# serpent utf-8 python2.6\n"); - serialize(obj, pw, 0); + sw.write("# serpent utf-8 python2.6\n"); + serialize(obj, sw, 0); - pw.flush(); + sw.flush(); final String ser = sw.toString(); - pw.close(); try { sw.close(); return ser.getBytes("utf-8"); @@ -107,7 +104,7 @@ public byte[] serialize(Object obj) } } - protected void serialize(Object obj, PrintWriter p, int level) + protected void serialize(Object obj, StringWriter sw, int level) { if(level>maximumLevel) throw new IllegalArgumentException("Object graph nesting too deep. Increase serializer.maximumLevel if you think you need more."); @@ -134,75 +131,75 @@ protected void serialize(Object obj, PrintWriter p, int level) // byte array? encode as base-64 if(componentType==Byte.TYPE) { - serialize_bytes((byte[])obj, p, level); + serialize_bytes((byte[])obj, sw, level); return; } else { - serialize_primitive_array(obj, p, level); + serialize_primitive_array(obj, sw, level); } return; } if(obj==null) { - p.print("None"); + sw.write("None"); } else if(obj instanceof String) { - serialize_string((String)obj, p, level); + serialize_string((String)obj, sw, level); } else if(type.isPrimitive() || isBoxed(type)) { - serialize_primitive(obj, p, level); + serialize_primitive(obj, sw, level); } else if(obj instanceof Enum) { - serialize_string(obj.toString(), p, level); + serialize_string(obj.toString(), sw, level); } else if(obj instanceof BigDecimal) { - serialize_bigdecimal((BigDecimal)obj, p, level); + serialize_bigdecimal((BigDecimal)obj, sw, level); } else if(obj instanceof Number) { - serialize_primitive(obj, p, level); + serialize_primitive(obj, sw, level); } else if(obj instanceof Date) { - serialize_date((Date)obj, p, level); + serialize_date((Date)obj, sw, level); } else if(obj instanceof Calendar) { - serialize_calendar((Calendar)obj, p, level); + serialize_calendar((Calendar)obj, sw, level); } else if(obj instanceof UUID) { - serialize_uuid((UUID)obj, p, level); + serialize_uuid((UUID)obj, sw, level); } else if(obj instanceof Set) { - serialize_set((Set)obj, p, level); + serialize_set((Set)obj, sw, level); } else if(obj instanceof Map) { - serialize_dict((Map)obj, p, level); + serialize_dict((Map)obj, sw, level); } else if(obj instanceof Collection) { - serialize_collection((Collection)obj, p, level); + serialize_collection((Collection)obj, sw, level); } else if(obj instanceof ComplexNumber) { - serialize_complex((ComplexNumber)obj, p, level); + serialize_complex((ComplexNumber)obj, sw, level); } else if(obj instanceof Exception) { - serialize_exception((Exception)obj, p, level); + serialize_exception((Exception)obj, sw, level); } else if(obj instanceof Serializable) { - serialize_class(obj, p, level); + serialize_class(obj, sw, level); } else { @@ -255,70 +252,70 @@ else if(classname.equals("org.python.core.PyMemoryView")) { throw new IllegalArgumentException("cannot serialize Jython object of type "+obj.getClass()); } - protected void serialize_collection(Collection collection, PrintWriter p, int level) + protected void serialize_collection(Collection collection, StringWriter sw, int level) { // output a list - p.print("["); - serialize_sequence_elements(collection, false, p, level+1); + sw.write("["); + serialize_sequence_elements(collection, false, sw, level+1); if(this.indent && collection.size()>0) { for(int i=0; i elts, boolean trailingComma, PrintWriter p, int level) + protected void serialize_sequence_elements(Collection elts, boolean trailingComma, StringWriter sw, int level) { if(elts.size()==0) return; int count=0; if(this.indent) { - p.print("\n"); + sw.write("\n"); String innerindent = ""; for(int i=0; i set, PrintWriter p, int level) + protected void serialize_set(Set set, StringWriter sw, int level) { if(!this.setliterals) { // output a tuple instead of a set-literal - serialize_tuple(set, p, level); + serialize_tuple(set, sw, level); return; } if(set.size()>0) { - p.print("{"); + sw.write("{"); Collection output = set; if(this.indent) { @@ -331,59 +328,59 @@ protected void serialize_set(Set set, PrintWriter p, int level) } output = outputset; } - serialize_sequence_elements(output, false, p, level+1); + serialize_sequence_elements(output, false, sw, level+1); if(this.indent) { for(int i=0; i items = new ArrayList(length); for(int i=0; i items, PrintWriter p, int level) + protected void serialize_tuple(Collection items, StringWriter sw, int level) { - p.print("("); - serialize_sequence_elements(items, items.size()==1, p, level+1); + sw.write("("); + serialize_sequence_elements(items, items.size()==1, sw, level+1); if(this.indent && items.size()>0) { for(int i=0; i dict = new HashMap(); dict.put("data", str); dict.put("encoding", "base64"); - serialize_dict(dict, p, level); + serialize_dict(dict, sw, level); } - protected void serialize_dict(Map dict, PrintWriter p,int level) + protected void serialize_dict(Map dict, StringWriter sw,int level) { if(dict.size()==0) { - p.print("{}"); + sw.write("{}"); return; } @@ -393,7 +390,7 @@ protected void serialize_dict(Map dict, PrintWriter p,int level) String innerindent = " "; for(int i=0; i outputdict = dict; @@ -405,66 +402,66 @@ protected void serialize_dict(Map dict, PrintWriter p,int level) for(Map.Entry e: outputdict.entrySet()) { - p.print(innerindent); - serialize(e.getKey(), p, level+1); - p.print(": "); - serialize(e.getValue(), p, level+1); + sw.write(innerindent); + serialize(e.getKey(), sw, level+1); + sw.write(": "); + serialize(e.getValue(), sw, level+1); counter++; if(counter e: dict.entrySet()) { - serialize(e.getKey(), p, level+1); - p.print(":"); - serialize(e.getValue(), p, level+1); + serialize(e.getKey(), sw, level+1); + sw.write(":"); + serialize(e.getValue(), sw, level+1); counter++; if(counter=0) - p.print("+"); - serialize_primitive(cplx.imaginary, p, level); - p.print("j)"); + sw.write("+"); + serialize_primitive(cplx.imaginary, sw, level); + sw.write("j)"); } - protected void serialize_uuid(UUID obj, PrintWriter p, int level) + protected void serialize_uuid(UUID obj, StringWriter sw, int level) { - serialize_string(obj.toString(), p, level); + serialize_string(obj.toString(), sw, level); } - protected void serialize_bigdecimal(BigDecimal decimal, PrintWriter p, int level) + protected void serialize_bigdecimal(BigDecimal decimal, StringWriter sw, int level) { - serialize_string(decimal.toEngineeringString(), p, level); + serialize_string(decimal.toEngineeringString(), sw, level); } private static final HashSet> boxedTypes; @@ -485,7 +482,7 @@ protected boolean isBoxed(Class type) return boxedTypes.contains(type); } - protected void serialize_class(Object obj, PrintWriter p, int level) + protected void serialize_class(Object obj, StringWriter sw, int level) { Map map; IClassSerializer converter=getCustomConverter(obj.getClass()); @@ -529,7 +526,7 @@ protected void serialize_class(Object obj, PrintWriter p, int level) throw new IllegalArgumentException("couldn't introspect javabean: "+e); } } - serialize_dict(map, p, level); + serialize_dict(map, sw, level); } protected IClassSerializer getCustomConverter(Class type) { @@ -549,16 +546,16 @@ protected IClassSerializer getCustomConverter(Class type) { return null; } - protected void serialize_primitive(Object obj, PrintWriter p, int level) + protected void serialize_primitive(Object obj, StringWriter sw, int level) { if(obj instanceof Boolean || obj.getClass()==Boolean.TYPE) { - p.print(obj.equals(Boolean.TRUE)? "True": "False"); + sw.write(obj.equals(Boolean.TRUE)? "True": "False"); } else if (obj instanceof Float || obj.getClass()==Float.TYPE) { Float f = (Float)obj; - serialize_primitive(f.doubleValue(), p, level); + serialize_primitive(f.doubleValue(), sw, level); } else if (obj instanceof Double || obj.getClass()==Double.TYPE) { @@ -566,21 +563,21 @@ else if (obj instanceof Double || obj.getClass()==Double.TYPE) if(d.isInfinite()) { // output a literal expression that overflows the float and results in +/-INF if(d>0.0) { - p.print("1e30000"); + sw.write("1e30000"); } else { - p.print("-1e30000"); + sw.write("-1e30000"); } } else if(d.isNaN()) { // there's no literal expression for a float NaN... - p.print("{'__class__':'float','value':'nan'}"); + sw.write("{'__class__':'float','value':'nan'}"); } else { - p.print(d); + sw.write(d.toString()); } } else { - p.print(obj); + sw.write(obj.toString()); } } @@ -608,7 +605,7 @@ else if(d.isNaN()) { repr_255[0xad] = "\\0xad"; } - protected void serialize_string(String str, PrintWriter p, int level) + protected void serialize_string(String str, StringWriter sw, int level) { // create a 'repr' string representation following the same escaping rules as python 3.x repr() does. StringBuilder b=new StringBuilder(str.length()); @@ -635,21 +632,21 @@ protected void serialize_string(String str, PrintWriter p, int level) if(!containsSingleQuote) { b.insert(0, '\''); b.append('\''); - p.print(b.toString()); + sw.write(b.toString()); } else if (!containsQuote) { b.insert(0, '"'); b.append('"'); - p.print(b.toString()); + sw.write(b.toString()); } else { String str2 = b.toString(); str2 = str2.replace("'", "\\'"); - p.print("'"); - p.print(str2); - p.print("'"); + sw.write("'"); + sw.write(str2); + sw.write("'"); } } - protected void serialize_exception(Exception ex, PrintWriter p, int level) + protected void serialize_exception(Exception ex, StringWriter sw, int level) { Map dict; IClassSerializer converter=classToDictRegistry.get(ex.getClass()); @@ -668,6 +665,6 @@ protected void serialize_exception(Exception ex, PrintWriter p, int level) dict.put("args", new String[]{ex.getMessage()}); dict.put("attributes", java.util.Collections.EMPTY_MAP); } - serialize_dict(dict, p, level); + serialize_dict(dict, sw, level); } } diff --git a/java/src/test/java/net/razorvine/serpent/test/CycleTest.java b/java/src/test/java/net/razorvine/serpent/test/CycleTest.java index d3f257d..6577e34 100644 --- a/java/src/test/java/net/razorvine/serpent/test/CycleTest.java +++ b/java/src/test/java/net/razorvine/serpent/test/CycleTest.java @@ -99,7 +99,7 @@ public void testClassCycle() public void testMaxLevel() { Serializer ser = new Serializer(); - assertEquals(1000, ser.maximumLevel); + assertEquals(800, ser.maximumLevel); Object[] array = new Object[] { "level1", diff --git a/java/src/test/java/net/razorvine/serpent/test/SerializeTest.java b/java/src/test/java/net/razorvine/serpent/test/SerializeTest.java index f3b1d8f..5f0043d 100644 --- a/java/src/test/java/net/razorvine/serpent/test/SerializeTest.java +++ b/java/src/test/java/net/razorvine/serpent/test/SerializeTest.java @@ -113,7 +113,7 @@ public void testStuff() assertEquals("'blerp'", S(result)); result = ser.serialize("\\"); result=strip_header(result); - assertEquals("'\\\\'", S(result)); // XXX + assertEquals("'\\\\'", S(result)); result = ser.serialize(UUID.fromString("f1f8d00e-49a5-4662-ac1d-d5f0426ed293")); result=strip_header(result); assertEquals("'f1f8d00e-49a5-4662-ac1d-d5f0426ed293'", S(result)); From d7e2720f0bd120e2a9126e30d2c466f8a189339b Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Thu, 16 Feb 2017 23:32:29 +0100 Subject: [PATCH 072/230] dotnet: rework unicode string repr --- dotnet/Serpent.Test/CycleTest.cs | 2 +- dotnet/Serpent.Test/SerializeTest.cs | 81 +++++++++++++++++++------ dotnet/Serpent/Serializer.cs | 89 ++++++++++++++++++++-------- 3 files changed, 128 insertions(+), 44 deletions(-) diff --git a/dotnet/Serpent.Test/CycleTest.cs b/dotnet/Serpent.Test/CycleTest.cs index fec62dd..baea64c 100644 --- a/dotnet/Serpent.Test/CycleTest.cs +++ b/dotnet/Serpent.Test/CycleTest.cs @@ -100,7 +100,7 @@ public void testClassCycle() public void testMaxLevel() { Serializer ser = new Serializer(); - Assert.AreEqual(1000, ser.MaximumLevel); + Assert.AreEqual(800, ser.MaximumLevel); Object[] array = new Object[] { "level1", diff --git a/dotnet/Serpent.Test/SerializeTest.cs b/dotnet/Serpent.Test/SerializeTest.cs index bfe2c1a..9609a68 100644 --- a/dotnet/Serpent.Test/SerializeTest.cs +++ b/dotnet/Serpent.Test/SerializeTest.cs @@ -101,27 +101,70 @@ public void TestStrings() } [Test] - public void TestUnicode() + public void TestUnicodeEscapes() { - Serializer serpent = new Serializer(); - byte[] ser = serpent.Serialize("euro\u20ac"); - byte[] data = strip_header(ser); - Assert.AreEqual(new byte[] {39, 101, 117, 114, 111, 0xe2, 0x82, 0xac, 39}, data); - - ser = serpent.Serialize("A\n\t\\Z"); - // 'A\\n\\t\\\\Z' (10 bytes) - data = strip_header(ser); - Assert.AreEqual(new byte[] {39, 65, 92, 110, 92, 116, 92, 92, 90, 39}, data); + Serializer serpent=new Serializer(); - ser = serpent.Serialize("euro\u20ac\nlastline\ttab\\@slash"); - // 'euro\xe2\x82\xac\\nlastline\\ttab\\\\@slash' (32 bytes) - data = strip_header(ser); - Assert.AreEqual(new byte[] { - 39, 101, 117, 114, 111, 226, 130, 172, - 92, 110, 108, 97, 115, 116, 108, 105, - 110, 101, 92, 116, 116, 97, 98, 92, - 92, 64, 115, 108, 97, 115, 104, 39} - , data); + // regular escaped chars first + byte[] ser = serpent.Serialize("\b\r\n\f\t \\"); + byte[] data = strip_header(ser); + // '\\x08\\r\\n\\x0c\\t \\\\' + Assert.AreEqual(new byte[] {39, + 92, 120, 48, 56, + 92, 114, + 92, 110, + 92, 120, 48, 99, + 92, 116, + 32, + 92, 92, + 39}, data); + + // simple cases (chars < 0x80) + ser = serpent.Serialize("\u0000\u0001\u001f\u007f"); + data = strip_header(ser); + // '\\x00\\x01\\x1f\\x7f' + Assert.AreEqual(new byte[] {39, + 92, 120, 48, 48, + 92, 120, 48, 49, + 92, 120, 49, 102, + 92, 120, 55, 102, + 39 }, data); + + // chars 0x80 .. 0xff + ser = serpent.Serialize("\u0080\u0081\u00ff"); + data = strip_header(ser); + // '\\x80\\x81\xc3\xbf' (has some utf-8 encoded chars in it) + Assert.AreEqual(new byte[] {39, + 92, 120, 56, 48, + 92, 120, 56, 49, + 195, 191, + 39}, data); + + // chars above 0xff + ser = serpent.Serialize("\u0100\u20ac\u8899"); + data = strip_header(ser); + // '\xc4\x80\xe2\x82\xac\xe8\xa2\x99' (has some utf-8 encoded chars in it) + Assert.AreEqual(new byte[] {39, 196, 128, 226, 130, 172, 232, 162, 153, 39}, data); + +// // some random high chars that are all printable in python and not escaped +// ser = serpent.Serialize("\u0377\u082d\u10c5\u135d\uac00"); +// data = strip_header(ser); +// Console.WriteLine(S(data)); // XXX +// // '\xcd\xb7\xe0\xa0\xad\xe1\x83\x85\xe1\x8d\x9d\xea\xb0\x80' (only a bunch of utf-8 encoded chars) +// Assert.AreEqual(new byte[] {39, 205, 183, 224, 160, 173, 225, 131, 133, 225, 141, 157, 234, 176, 128, 39}, data); + + // some random high chars that are all non-printable in python and that are escaped + ser = serpent.Serialize("\u0378\u082e\u10c6\u135c\uabff"); + data = strip_header(ser); + Console.WriteLine(S(data)); // XXX + // '\\u0378\\u082e\\u10c6\\u135c\\uabff' + Assert.AreEqual(new byte[] {39, + 92, 117, 48, 51, 55, 56, + 92, 117, 48, 56, 50, 101, + 92, 117, 49, 48, 99, 54, + 92, 117, 49, 51, 53, 99, + 92, 117, 97, 98, 102, 102, + 39}, data); } [Test] diff --git a/dotnet/Serpent/Serializer.cs b/dotnet/Serpent/Serializer.cs index 6f1b165..951cea9 100644 --- a/dotnet/Serpent/Serializer.cs +++ b/dotnet/Serpent/Serializer.cs @@ -44,7 +44,7 @@ public class Serializer /// This limit has been set to avoid troublesome stack overflow errors. /// (If it is reached, an IllegalArgumentException is thrown instead with a clear message) /// - public int MaximumLevel = 1000; // avoids stackoverflow errors + public int MaximumLevel = 800; // avoids stackoverflow errors private static IDictionary> classToDictRegistry = new Dictionary>(); @@ -75,15 +75,13 @@ public static void RegisterClass(Type clazz, Func converter /// public byte[] Serialize(object obj) { - using(MemoryStream ms = new MemoryStream()) + using(MemoryStream ms = new MemoryStream()) // XXX StringWriter? using(TextWriter tw = new StreamWriter(ms, new UTF8Encoding(false))) // don't write BOM { - string header = "# serpent utf-8 "; if(this.SetLiterals) - header += "python3.2\n"; //set-literals require python 3.2+ to deserialize (ast.literal_eval limitation) + tw.Write("# serpent utf-8 python3.2\n"); //set-literals require python 3.2+ to deserialize (ast.literal_eval limitation) else - header += "python2.6\n"; - tw.Write(header); + tw.Write("# serpent utf-8 python2.6\n"); Serialize(obj, tw, 0); tw.Flush(); return ms.ToArray(); @@ -343,27 +341,70 @@ protected void Serialize_bytes(byte[] data, TextWriter tw, int level) Serialize_dict(dict, tw, level); } + + // the repr translation table for characters 0x00-0xff + private static readonly string[] repr_255; + static Serializer() { + repr_255=new String[256]; + for(int c=0; c<32; ++c) { + repr_255[c] = "\\x"+c.ToString("x2"); + } + for(int c=0x20; c<0x7f; ++c) { + repr_255[c] = Convert.ToString((char)c); + } + for(int c=0x7f; c<=0xa0; ++c) { + repr_255[c] = "\\x"+c.ToString("x2"); + } + for(int c=0xa1; c<=0xff; ++c) { + repr_255[c] = Convert.ToString((char)c); + } + // odd ones out: + repr_255['\t'] = "\\t"; + repr_255['\n'] = "\\n"; + repr_255['\r'] = "\\r"; + repr_255['\\'] = "\\\\"; + repr_255[0xad] = "\\0xad"; + } + protected void Serialize_string(string str, TextWriter tw, int level) { - // backslash-escaped string - str = str.Replace("\\", "\\\\"); // double-escape the backslashes - str = str.Replace("\a", "\\a"); - str = str.Replace("\b", "\\b"); - str = str.Replace("\f", "\\f"); - str = str.Replace("\n", "\\n"); - str = str.Replace("\r", "\\r"); - str = str.Replace("\t", "\\t"); - str = str.Replace("\v", "\\v"); - if(!str.Contains("'")) - str = "'" + str + "'"; - else if(!str.Contains("\"")) - str = '"' + str + '"'; - else - { - str = str.Replace("'", "\\'"); - str = "'" + str + "'"; + // create a 'repr' string representation following the same escaping rules as python 3.x repr() does. + StringBuilder b=new StringBuilder(str.Length); + bool containsSingleQuote=false; + bool containsQuote=false; + foreach(char c in str) + { + containsSingleQuote |= c=='\''; + containsQuote |= c=='"'; + + if(c<256) { + // characters 0..255 via quick lookup table + b.Append(repr_255[c]); + } else { + if(Char.IsLetterOrDigit(c) || Char.IsNumber(c) || Char.IsPunctuation(c) || Char.IsSymbol(c)) { + b.Append(c); + } else { + b.Append("\\u"); + b.Append(((int)c).ToString("x4")); + } + } + } + + if(!containsSingleQuote) { + b.Insert(0, '\''); + b.Append('\''); + tw.Write(b.ToString()); + } else if (!containsQuote) { + b.Insert(0, '"'); + b.Append('"'); + tw.Write(b.ToString()); + } else { + String str2 = b.ToString(); + str2 = str2.Replace("'", "\\'"); + tw.Write("'"); + tw.Write(str2); + tw.Write("'"); } - tw.Write(str); } protected void Serialize_datetime(DateTime dt, TextWriter tw, int level) From 4ade43db2e58810057da0a0bb5cffb4869d236dd Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Thu, 16 Feb 2017 23:35:08 +0100 Subject: [PATCH 073/230] dotnet: optimized serializer io a bit --- dotnet/Serpent/Serializer.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dotnet/Serpent/Serializer.cs b/dotnet/Serpent/Serializer.cs index 951cea9..d96bb97 100644 --- a/dotnet/Serpent/Serializer.cs +++ b/dotnet/Serpent/Serializer.cs @@ -75,8 +75,7 @@ public static void RegisterClass(Type clazz, Func converter /// public byte[] Serialize(object obj) { - using(MemoryStream ms = new MemoryStream()) // XXX StringWriter? - using(TextWriter tw = new StreamWriter(ms, new UTF8Encoding(false))) // don't write BOM + using(StringWriter tw = new StringWriter()) { if(this.SetLiterals) tw.Write("# serpent utf-8 python3.2\n"); //set-literals require python 3.2+ to deserialize (ast.literal_eval limitation) @@ -84,7 +83,7 @@ public byte[] Serialize(object obj) tw.Write("# serpent utf-8 python2.6\n"); Serialize(obj, tw, 0); tw.Flush(); - return ms.ToArray(); + return Encoding.UTF8.GetBytes(tw.ToString()); } } From 5d2364d0686f6de8a0b6b145df726f836199bc7b Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Thu, 16 Feb 2017 23:37:08 +0100 Subject: [PATCH 074/230] dotnet: remove debug print --- dotnet/Serpent.Test/SerializeTest.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/dotnet/Serpent.Test/SerializeTest.cs b/dotnet/Serpent.Test/SerializeTest.cs index 9609a68..cb99ac3 100644 --- a/dotnet/Serpent.Test/SerializeTest.cs +++ b/dotnet/Serpent.Test/SerializeTest.cs @@ -156,7 +156,6 @@ public void TestUnicodeEscapes() // some random high chars that are all non-printable in python and that are escaped ser = serpent.Serialize("\u0378\u082e\u10c6\u135c\uabff"); data = strip_header(ser); - Console.WriteLine(S(data)); // XXX // '\\u0378\\u082e\\u10c6\\u135c\\uabff' Assert.AreEqual(new byte[] {39, 92, 117, 48, 51, 55, 56, From d8eb03414db9d87b1b367f049e4a3d06997fc8ce Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Fri, 17 Feb 2017 01:33:02 +0100 Subject: [PATCH 075/230] java fix in string repr and unittest for it --- .../java/net/razorvine/serpent/Parser.java | 6 ++++++ .../java/net/razorvine/serpent/Serializer.java | 6 +++--- .../razorvine/serpent/test/SerializeTest.java | 18 ++++++++++++++++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/java/src/main/java/net/razorvine/serpent/Parser.java b/java/src/main/java/net/razorvine/serpent/Parser.java index 32c1597..662ffc8 100644 --- a/java/src/main/java/net/razorvine/serpent/Parser.java +++ b/java/src/main/java/net/razorvine/serpent/Parser.java @@ -557,6 +557,12 @@ PrimitiveNode parseString(SeekableStringReader sr) case 't': sb.append('\t'); break; + case 'x': // "\x00" + sb.append((char)Integer.parseInt(sr.read(2), 16)); + break; + case 'u': // "\u0000" + sb.append((char)Integer.parseInt(sr.read(4), 16)); + break; default: sb.append(c); break; diff --git a/java/src/main/java/net/razorvine/serpent/Serializer.java b/java/src/main/java/net/razorvine/serpent/Serializer.java index d81186f..329432f 100644 --- a/java/src/main/java/net/razorvine/serpent/Serializer.java +++ b/java/src/main/java/net/razorvine/serpent/Serializer.java @@ -602,13 +602,13 @@ else if(d.isNaN()) { repr_255['\n'] = "\\n"; repr_255['\r'] = "\\r"; repr_255['\\'] = "\\\\"; - repr_255[0xad] = "\\0xad"; + repr_255[0xad] = "\\xad"; } protected void serialize_string(String str, StringWriter sw, int level) { // create a 'repr' string representation following the same escaping rules as python 3.x repr() does. - StringBuilder b=new StringBuilder(str.length()); + StringBuilder b=new StringBuilder(str.length()*2); boolean containsSingleQuote=false; boolean containsQuote=false; for(int i=0; i chars64k.length); + + Parser p=new Parser(); + String result = (String)p.parse(data).getData(); + assertEquals(str64k, result); + } + @Test public void testNullByte() { From 8fa229452167b3f9169db390ba84daad404f45ef Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Fri, 17 Feb 2017 01:56:42 +0100 Subject: [PATCH 076/230] dotnet: rewrote serialization and parsing of strings --- dotnet/Serpent.Test/ParserTest.cs | 18 ++++++++++++++++++ dotnet/Serpent/Parser.cs | 6 ++++++ dotnet/Serpent/Serializer.cs | 4 ++-- dotnet/Serpent/Serpent.nuspec | 1 + 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/dotnet/Serpent.Test/ParserTest.cs b/dotnet/Serpent.Test/ParserTest.cs index 6596b89..9f27521 100644 --- a/dotnet/Serpent.Test/ParserTest.cs +++ b/dotnet/Serpent.Test/ParserTest.cs @@ -506,6 +506,24 @@ public void TestUnicode() Assert.AreEqual(new Ast.StringNode(value), p.Parse(bytes).Root); } + [Test] + public void TestLongUnicodeRoundtrip() + { + Char[] chars64k = new Char[65536]; + for(int i=0; i<=65535; ++i) + chars64k[i]=(Char)i; + + String str64k= new String(chars64k); + + Serializer ser=new Serializer(); + byte[] data = ser.Serialize(str64k); + Assert.Greater(data.Length, chars64k.Length); + + Parser p=new Parser(); + String result = (String)p.Parse(data).GetData(); + Assert.AreEqual(str64k, result); + } + [Test] public void TestWhitespace() { diff --git a/dotnet/Serpent/Parser.cs b/dotnet/Serpent/Parser.cs index 98673f9..6c4851b 100644 --- a/dotnet/Serpent/Parser.cs +++ b/dotnet/Serpent/Parser.cs @@ -570,6 +570,12 @@ Ast.PrimitiveNode ParseString(SeekableStringReader sr) case 'v': sb.Append('\v'); break; + case 'x': // "\x00" + sb.Append((char)int.Parse(sr.Read(2), NumberStyles.HexNumber)); + break; + case 'u': // "\u0000" + sb.Append((char)int.Parse(sr.Read(4), NumberStyles.HexNumber)); + break; default: sb.Append(c); break; diff --git a/dotnet/Serpent/Serializer.cs b/dotnet/Serpent/Serializer.cs index d96bb97..a6f49c0 100644 --- a/dotnet/Serpent/Serializer.cs +++ b/dotnet/Serpent/Serializer.cs @@ -362,13 +362,13 @@ static Serializer() { repr_255['\n'] = "\\n"; repr_255['\r'] = "\\r"; repr_255['\\'] = "\\\\"; - repr_255[0xad] = "\\0xad"; + repr_255[0xad] = "\\xad"; } protected void Serialize_string(string str, TextWriter tw, int level) { // create a 'repr' string representation following the same escaping rules as python 3.x repr() does. - StringBuilder b=new StringBuilder(str.Length); + StringBuilder b=new StringBuilder(str.Length*2); bool containsSingleQuote=false; bool containsQuote=false; foreach(char c in str) diff --git a/dotnet/Serpent/Serpent.nuspec b/dotnet/Serpent/Serpent.nuspec index 07e1d20..cdfb2d8 100644 --- a/dotnet/Serpent/Serpent.nuspec +++ b/dotnet/Serpent/Serpent.nuspec @@ -19,6 +19,7 @@ such a Python expression itself, back into the equivalent .NET datatypes. More info for the Python version is on Pypi: https://pypi.python.org/pypi/serpent Serpent Python literal expression serialization +CRITICAL FIX: rewrote serialization and parsing of strings containing chars above 255, this didn't work. Now mimics python's repr(str) form as closely as possible. Added Serializer.MaximumLevel to avoid too deep recursion resulting in stack overflow errors. Copyright 2016 From ae5ff7d3e2c8ea786fe558847ef9eefcdac4b420 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Fri, 17 Feb 2017 02:41:39 +0100 Subject: [PATCH 077/230] python Serializer also got maximum_level to avoid recursion overflow --- serpent.py | 3 +++ tests/test_serpent.py | 26 +++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/serpent.py b/serpent.py index 8da6205..e02c462 100644 --- a/serpent.py +++ b/serpent.py @@ -281,6 +281,7 @@ def __init__(self, indent=False, set_literals=can_use_set_literals, module_in_cl self.module_in_classname = module_in_classname self.serialized_obj_ids = set() self.special_classes_registry_copy = None + self.maximum_level = min(sys.getrecursionlimit() // 3.5, 1000) def serialize(self, obj): """Serialize the object tree to bytes.""" @@ -307,6 +308,8 @@ def serialize(self, obj): _shortcut_dispatch_types = frozenset([float, complex, tuple, list, dict, set, frozenset]) def _serialize(self, obj, out, level): + if level > self.maximum_level: + raise ValueError("Object graph nesting too deep. Increase serializer.maximum_level if you think you need more.") t = type(obj) if t in _translate_types: obj = _translate_types[t](obj) diff --git a/tests/test_serpent.py b/tests/test_serpent.py index ed6e35f..358ebeb 100644 --- a/tests/test_serpent.py +++ b/tests/test_serpent.py @@ -6,7 +6,6 @@ """ from __future__ import print_function, division import sys -import ast import timeit import datetime import uuid @@ -932,6 +931,31 @@ def testClassCycle(self): serpent.dumps(d) self.assertEqual("Circular reference detected (class)", str(e.exception)) + # noinspection PyUnreachableCode + def testMaxLevel(self): + ser = serpent.Serializer() + self.assertGreater(ser.maximum_level, 100) + array=[] + arr=array + for level in range(min(sys.getrecursionlimit()+10, 2000)): + arr.append("level"+str(level)) + arr2 = [] + arr.append(arr2) + arr=arr2 + with self.assertRaises(ValueError) as x: + ser.serialize(array) + self.fail("should crash") + self.assertTrue("too deep" in str(x.exception)) + # check setting the maxlevel + array = ["level1", ["level2", ["level3", ["level4"]]]] + ser.maximum_level = 4 + ser.serialize(array) # should work + ser.maximum_level = 3 + with self.assertRaises(ValueError) as x: + ser.serialize(array) # should crash + self.fail("should crash") + self.assertTrue("too deep" in str(x.exception)) + class Cycle(object): def __init__(self): From 9c9417bdc32f8ecb50ef62628a4ad06e82f1623e Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Fri, 17 Feb 2017 02:48:38 +0100 Subject: [PATCH 078/230] pypy has low recursionlimit --- tests/test_serpent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_serpent.py b/tests/test_serpent.py index 358ebeb..19292cd 100644 --- a/tests/test_serpent.py +++ b/tests/test_serpent.py @@ -934,7 +934,7 @@ def testClassCycle(self): # noinspection PyUnreachableCode def testMaxLevel(self): ser = serpent.Serializer() - self.assertGreater(ser.maximum_level, 100) + self.assertGreater(ser.maximum_level, 20) # 20 seems quite low, but Pypy appears to have a low default recursionlimit (100) array=[] arr=array for level in range(min(sys.getrecursionlimit()+10, 2000)): From d424769dacfc4464c815d8106b80ca1497e7ba67 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Fri, 17 Feb 2017 21:35:34 +0100 Subject: [PATCH 079/230] added 65535-char serialization test --- dotnet/Serpent.Test/SerializeTest.cs | 2 +- .../razorvine/serpent/test/ParserTest.java | 18 ++++++++++++++++++ .../razorvine/serpent/test/SerializeTest.java | 18 ------------------ tests/test_serpent.py | 19 +++++++++++++++++++ 4 files changed, 38 insertions(+), 19 deletions(-) diff --git a/dotnet/Serpent.Test/SerializeTest.cs b/dotnet/Serpent.Test/SerializeTest.cs index cb99ac3..367538d 100644 --- a/dotnet/Serpent.Test/SerializeTest.cs +++ b/dotnet/Serpent.Test/SerializeTest.cs @@ -710,7 +710,7 @@ public void testAbstractBaseClassHierarchyPickler() } [Test] - public void testInterfaceHierarchyPickler() + public void TestInterfaceHierarchyPickler() { BaseClassWithInterface b = new BaseClassWithInterface(); SubClassWithInterface sub = new SubClassWithInterface(); diff --git a/java/src/test/java/net/razorvine/serpent/test/ParserTest.java b/java/src/test/java/net/razorvine/serpent/test/ParserTest.java index 493bfe0..92a05fa 100644 --- a/java/src/test/java/net/razorvine/serpent/test/ParserTest.java +++ b/java/src/test/java/net/razorvine/serpent/test/ParserTest.java @@ -540,6 +540,24 @@ public void TestUnicode() throws UnsupportedEncodingException assertEquals(new StringNode(value), p.parse(str).root); assertEquals(new StringNode(value), p.parse(bytes).root); } + + @Test + public void testLongUnicodeRoundtrip() + { + char[] chars64k = new char[65536]; + for(int i=0; i<=65535; ++i) + chars64k[i]=(char)i; + + String str64k= new String(chars64k); + + Serializer ser=new Serializer(); + byte[] data = ser.serialize(str64k); + assertTrue(data.length > chars64k.length); + + Parser p=new Parser(); + String result = (String)p.parse(data).getData(); + assertEquals(str64k, result); + } @Test public void TestWhitespace() diff --git a/java/src/test/java/net/razorvine/serpent/test/SerializeTest.java b/java/src/test/java/net/razorvine/serpent/test/SerializeTest.java index 0eace28..5f0043d 100644 --- a/java/src/test/java/net/razorvine/serpent/test/SerializeTest.java +++ b/java/src/test/java/net/razorvine/serpent/test/SerializeTest.java @@ -213,24 +213,6 @@ public void testUnicodeEscapes() 39}, data); } - @Test - public void testLongUnicodeRoundtrip() - { - char[] chars64k = new char[65536]; - for(int i=0; i<=65535; ++i) - chars64k[i]=(char)i; - - String str64k= new String(chars64k); - - Serializer ser=new Serializer(); - byte[] data = ser.serialize(str64k); - assertTrue(data.length > chars64k.length); - - Parser p=new Parser(); - String result = (String)p.parse(data).getData(); - assertEquals(str64k, result); - } - @Test public void testNullByte() { diff --git a/tests/test_serpent.py b/tests/test_serpent.py index 19292cd..d3da638 100644 --- a/tests/test_serpent.py +++ b/tests/test_serpent.py @@ -5,7 +5,9 @@ Software license: "MIT software license". See http://opensource.org/licenses/MIT """ from __future__ import print_function, division +import __future__ import sys +import ast import timeit import datetime import uuid @@ -227,6 +229,23 @@ def test_unicode(self): data = serpent.loads(ser) self.assertEqual(u, data) + @unittest.skipIf(os.name=="java", "Jython can't deserialize strings with unicode in them, bug #2008") + def test_unicode_escape_allchars(self): + # this checks for all 0x0000-0xffff chars that they will be serialized + # into a proper repr form and when processed back by ast.literal_parse directly + # will get turned back into the chars 0x0000-0xffff again + all_chars = u"".join(unichr(c) for c in range(65536)) + ser = serpent.dumps(all_chars) + self.assertGreater(len(ser), len(all_chars)) + ser = ser.decode("utf-8") + if sys.version_info < (3, 0): + ser = compile(ser, "", mode="eval", flags=ast.PyCF_ONLY_AST | __future__.unicode_literals.compiler_flag) + data = ast.literal_eval(ser) + self.assertEqual(65536, len(data)) + for i, c in enumerate(data): + if unichr(i) != c: + self.fail("char different for "+str(i)) + def test_unicode_quotes(self): ser = serpent.dumps(unicode("quotes'\"")) data = strip_header(ser) From 7269b41a1aa2e1d6ed75b71ade495c3bcb0881b4 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Fri, 17 Feb 2017 21:45:46 +0100 Subject: [PATCH 080/230] [maven-release-plugin] prepare release serpent-1.17 --- java/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index f5a0c09..fbb9a1c 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -9,7 +9,7 @@ net.razorvine serpent - 1.17-SNAPSHOT + 1.17 jar serpent @@ -67,7 +67,7 @@ It is an alternative to JSON to provide easy data integration between Java and P https://github.com/irmen/Serpent scm:git:https://github.com/irmen/Serpent.git scm:git:https://github.com/irmen/Serpent.git - HEAD + serpent-1.17 Github From ae1710aba5678bd6279217b8df1f0da574e709a8 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Fri, 17 Feb 2017 21:45:53 +0100 Subject: [PATCH 081/230] [maven-release-plugin] prepare for next development iteration --- java/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index fbb9a1c..256273d 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -9,7 +9,7 @@ net.razorvine serpent - 1.17 + 1.18-SNAPSHOT jar serpent @@ -67,7 +67,7 @@ It is an alternative to JSON to provide easy data integration between Java and P https://github.com/irmen/Serpent scm:git:https://github.com/irmen/Serpent.git scm:git:https://github.com/irmen/Serpent.git - serpent-1.17 + HEAD Github From 039b417345271722b1a3a5d17b1d3cae7f158d6d Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Fri, 17 Feb 2017 21:55:11 +0100 Subject: [PATCH 082/230] copyright year bump --- LICENSE | 2 +- dotnet/Serpent/Serpent.nuspec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index abfce8a..0c50aed 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2016 Irmen de Jong +Copyright (c) 2017 Irmen de Jong Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/dotnet/Serpent/Serpent.nuspec b/dotnet/Serpent/Serpent.nuspec index cdfb2d8..ec4948a 100644 --- a/dotnet/Serpent/Serpent.nuspec +++ b/dotnet/Serpent/Serpent.nuspec @@ -22,7 +22,7 @@ More info for the Python version is on Pypi: https://pypi.python.org/pypi/serpen CRITICAL FIX: rewrote serialization and parsing of strings containing chars above 255, this didn't work. Now mimics python's repr(str) form as closely as possible. Added Serializer.MaximumLevel to avoid too deep recursion resulting in stack overflow errors. - Copyright 2016 + Copyright 2017 serialization python pyro From 62c01b39da48e0c70b1e56db168c30f409a99b62 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Fri, 17 Feb 2017 22:32:00 +0100 Subject: [PATCH 083/230] workaround for jython unicode parse bug --- serpent.py | 27 +++++++-------------------- tests/test_serpent.py | 23 +++++++++++++++-------- 2 files changed, 22 insertions(+), 28 deletions(-) diff --git a/serpent.py b/serpent.py index e02c462..9983ff8 100644 --- a/serpent.py +++ b/serpent.py @@ -88,28 +88,15 @@ def loads(serialized_bytes): raise ValueError("The serpent data contains 0-bytes so it cannot be parsed by ast.literal_eval. Has it been corrupted?") serialized = serialized_bytes.decode("utf-8") if sys.version_info < (3, 0): + # python 2.x: parse with unicode_literals (promotes all strings to unicode) + serialized = compile(serialized, "", mode="eval", flags=ast.PyCF_ONLY_AST | __future__.unicode_literals.compiler_flag) if os.name == "java": - # Because of a bug in Jython we cannot correctly parse strings with characters >255 in them. - # These are encoded using the \u???? escape sequence, and should be parsed back into a proper - # unicode string. However the ast module in Jython is unable to do this correctly. + # The ast module in Jython will not have parsed this correctly into unicode literals. + # So we have to patch up the ast tree ourselves and decode Str nodes to unicode manually. # See http://bugs.jython.org/issue2008 - # I tried various things (unicode-escaping, re-encoding in utf-8, modifying the ast syntax tree to - # try to convert the str nodes, but all seems in vain. The parsing simply processes the unicode wrong, - # and loses information in the process that cannot be restored by postprocessing. - # The only option for now seems to be to give up if we encounter such a string. - # So for Jython: only use ASCII strings or encode/decode the unicode strings yourself into bytes or ASCII - # before sending them through Serpent. - import re - if re.search(r"\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}", serialized): - raise ValueError("Unsupported serialized string encountered. Unfortunately it is currently " \ - "impossible to deserialize a string with characters above value 255 in it, " \ - "due to a bug with the ast module in Jython. Either change the string that is " \ - "given to this application (for instance make sure it is ASCII only), " \ - "or wait until the bug in Jython is fixed: http://bugs.jython.org/issue2008 "\ - "(Unfortunately there seems little progress in solving the bug)") - else: - # python 2.x: parse with unicode_literals (promotes all strings to unicode) - serialized = compile(serialized, "", mode="eval", flags=ast.PyCF_ONLY_AST | __future__.unicode_literals.compiler_flag) + for node in ast.walk(serialized): + if isinstance(node, ast.Str): + node.s = node.s.decode("unicode-escape") try: if os.name != "java" and sys.platform != "cli": gc.disable() diff --git a/tests/test_serpent.py b/tests/test_serpent.py index d3da638..d901297 100644 --- a/tests/test_serpent.py +++ b/tests/test_serpent.py @@ -48,7 +48,6 @@ def test_deserialize(self): data = serpent.loads(b"555") self.assertEqual(555, data) - @unittest.skipIf(os.name=="java", "Jython can't deserialize strings with unicode in them, bug #2008") def test_deserialize_unichr(self): unicodestring = "euro" + unichr(0x20ac) encoded = repr(unicodestring).encode("utf-8") @@ -77,7 +76,6 @@ def test_trailing_comma_set(self): v = serpent.loads(b"{1,2,3,}") self.assertEqual(set([1, 2, 3]), v) - @unittest.skipIf(os.name=="java", "Jython can't deserialize strings with unicode in them, bug #2008") def test_unicode_escapes(self): v = serpent.loads(b"'\u20ac'") self.assertEqual(u"\u20ac", v) @@ -221,27 +219,37 @@ def test_detectNullByte(self): self.fail("must fail") self.assertTrue("0-bytes" in str(ex.exception)) - @unittest.skipIf(os.name=="java", "Jython can't deserialize strings with unicode in them, bug #2008") - def test_unicode(self): + @unittest.skipIf(os.name == "java", "jython can't parse unicode U's") + def test_unicode_U(self): u = "euro" + unichr(0x20ac)+"\U00022001" self.assertTrue(type(u) is unicode) ser = serpent.dumps(u) data = serpent.loads(ser) self.assertEqual(u, data) - @unittest.skipIf(os.name=="java", "Jython can't deserialize strings with unicode in them, bug #2008") def test_unicode_escape_allchars(self): # this checks for all 0x0000-0xffff chars that they will be serialized # into a proper repr form and when processed back by ast.literal_parse directly # will get turned back into the chars 0x0000-0xffff again - all_chars = u"".join(unichr(c) for c in range(65536)) + # For Jython we take the range upto 0x4100 because after that it starts + # complaining about surrogates or "mark invalid" (an utf-8 parsing bug it seems) + highest_char = 0x4100 if os.name == "java" else 0xffff + all_chars = u"".join(unichr(c) for c in range(highest_char+1)) ser = serpent.dumps(all_chars) self.assertGreater(len(ser), len(all_chars)) ser = ser.decode("utf-8") if sys.version_info < (3, 0): ser = compile(ser, "", mode="eval", flags=ast.PyCF_ONLY_AST | __future__.unicode_literals.compiler_flag) + if os.name == "java": + # The ast module in Jython will not have parsed this correctly into unicode literals. + # So we have to patch up the ast tree ourselves and decode Str nodes to unicode manually. + # (this is the same what Serpent does internally so we replicate it here) + # See http://bugs.jython.org/issue2008 + for node in ast.walk(ser): + if isinstance(node, ast.Str): + node.s = node.s.decode("unicode-escape") data = ast.literal_eval(ser) - self.assertEqual(65536, len(data)) + self.assertEqual(highest_char+1, len(data)) for i, c in enumerate(data): if unichr(i) != c: self.fail("char different for "+str(i)) @@ -272,7 +280,6 @@ def test_unicode_with_escapes_py2(self): d = strip_header(ser) self.assertEqual(b"'\\x07'", d) - @unittest.skipIf(os.name=="java", "Jython can't deserialize strings with unicode in them, bug #2008") @unittest.skipIf(sys.version_info >= (3, 0), "py2 escaping tested") def test_unicode_with_escapes_unichrs(self): ser = serpent.dumps("\a"+unichr(0x20ac)) From 794dfee81a148dc5b95137b23d7f76d907f1d416 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Fri, 17 Feb 2017 23:51:57 +0100 Subject: [PATCH 084/230] max_level reduced to 500, as 800 was still too high for certain systems --- java/src/main/java/net/razorvine/serpent/Serializer.java | 2 +- java/src/test/java/net/razorvine/serpent/test/CycleTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/java/src/main/java/net/razorvine/serpent/Serializer.java b/java/src/main/java/net/razorvine/serpent/Serializer.java index 329432f..cbb7ce6 100644 --- a/java/src/main/java/net/razorvine/serpent/Serializer.java +++ b/java/src/main/java/net/razorvine/serpent/Serializer.java @@ -34,7 +34,7 @@ public class Serializer * This limit has been set to avoid troublesome stack overflow errors. * (If it is reached, an IllegalArgumentException is thrown instead with a clear message) */ - public int maximumLevel = 800; // to avoid stack overflow errors + public int maximumLevel = 500; // to avoid stack overflow errors /** * Indent the resulting serpent serialization text? diff --git a/java/src/test/java/net/razorvine/serpent/test/CycleTest.java b/java/src/test/java/net/razorvine/serpent/test/CycleTest.java index 6577e34..ff2c945 100644 --- a/java/src/test/java/net/razorvine/serpent/test/CycleTest.java +++ b/java/src/test/java/net/razorvine/serpent/test/CycleTest.java @@ -99,7 +99,7 @@ public void testClassCycle() public void testMaxLevel() { Serializer ser = new Serializer(); - assertEquals(800, ser.maximumLevel); + assertEquals(500, ser.maximumLevel); Object[] array = new Object[] { "level1", From daf6ee579987db7f25c2ad1c5712c704ca896ded Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sat, 18 Feb 2017 14:53:44 +0100 Subject: [PATCH 085/230] perf tweaks --- serpent.py | 113 ++++++++++++++++++++----------------------- tests/performance.py | 12 +++-- 2 files changed, 61 insertions(+), 64 deletions(-) diff --git a/serpent.py b/serpent.py index 9983ff8..d1a77f4 100644 --- a/serpent.py +++ b/serpent.py @@ -66,7 +66,7 @@ import array import math -__version__ = "1.17" +__version__ = "1.18" __all__ = ["dump", "dumps", "load", "loads", "register_class", "unregister_class", "tobytes"] can_use_set_literals = sys.version_info >= (3, 2) # check if we can use set literals @@ -189,13 +189,6 @@ def from_buffer(data): return BytesWrapper(data) -if sys.version_info < (3, 0): - _repr = repr # python <3.0 won't need explicit encoding to utf-8, so we optimize this -else: - def _repr(obj): - return repr(obj).encode("utf-8") - - _repr_types = set([ str, int, @@ -278,7 +271,7 @@ def serialize(self, obj): header += "python3.2\n" # set-literals require python 3.2+ to deserialize (ast.literal_eval limitation) else: header += "python2.6\n" # don't change this even though we don't support 2.6 any longer, otherwise we can't read older serpent strings - out = [header.encode("utf-8")] + out = [header] try: if os.name != "java" and sys.platform != "cli": gc.disable() @@ -288,9 +281,7 @@ def serialize(self, obj): gc.enable() self.special_classes_registry_copy = None del self.serialized_obj_ids - if sys.platform == "cli": - return "".join(out) - return b"".join(out) + return "".join(out).encode("utf-8") _shortcut_dispatch_types = frozenset([float, complex, tuple, list, dict, set, frozenset]) @@ -302,7 +293,7 @@ def _serialize(self, obj, out, level): obj = _translate_types[t](obj) t = type(obj) if t in _repr_types: - out.append(_repr(obj)) # just a simple repr() is enough for these objects + out.append(repr(obj)) # just a simple repr() is enough for these objects return if t in self._shortcut_dispatch_types: # we shortcut these builtins directly to the dispatch function to avoid type lookup overhead below @@ -335,24 +326,24 @@ def ser_builtins_str(self, str_obj, out, level): def ser_builtins_float(self, float_obj, out, level): if math.isnan(float_obj): # there's no literal expression for a float NaN... - out.append(b"{'__class__':'float','value':'nan'}") + out.append("{'__class__':'float','value':'nan'}") elif math.isinf(float_obj): # output a literal expression that overflows the float and results in +/-INF if float_obj > 0: - out.append(b"1e30000") + out.append("1e30000") else: - out.append(b"-1e30000") + out.append("-1e30000") else: - out.append(repr(float_obj).encode("ascii")) + out.append(repr(float_obj)) dispatch[float] = ser_builtins_float def ser_builtins_complex(self, complex_obj, out, level): - out.append(b"(") + out.append("(") self.ser_builtins_float(complex_obj.real, out, level) if complex_obj.imag >= 0: - out.append(b"+") + out.append("+") self.ser_builtins_float(complex_obj.imag, out, level) - out.append(b"j)") + out.append("j)") dispatch[complex] = ser_builtins_complex if sys.version_info < (3, 0): @@ -374,25 +365,25 @@ def ser_builtins_tuple(self, tuple_obj, out, level): append = out.append serialize = self._serialize if self.indent and tuple_obj: - indent_chars = b" " * level - indent_chars_inside = indent_chars + b" " - append(b"(\n") + indent_chars = " " * level + indent_chars_inside = indent_chars + " " + append("(\n") for elt in tuple_obj: append(indent_chars_inside) serialize(elt, out, level + 1) - append(b",\n") + append(",\n") out[-1] = out[-1].rstrip() # remove the last \n if len(tuple_obj) > 1: del out[-1] # undo the last , - append(b"\n" + indent_chars + b")") + append("\n" + indent_chars + ")") else: - append(b"(") + append("(") for elt in tuple_obj: serialize(elt, out, level + 1) - append(b",") + append(",") if len(tuple_obj) > 1: del out[-1] # undo the last , - append(b")") + append(")") dispatch[tuple] = ser_builtins_tuple def ser_builtins_list(self, list_obj, out, level): @@ -402,23 +393,23 @@ def ser_builtins_list(self, list_obj, out, level): append = out.append serialize = self._serialize if self.indent and list_obj: - indent_chars = b" " * level - indent_chars_inside = indent_chars + b" " - append(b"[\n") + indent_chars = " " * level + indent_chars_inside = indent_chars + " " + append("[\n") for elt in list_obj: append(indent_chars_inside) serialize(elt, out, level + 1) - append(b",\n") + append(",\n") del out[-1] # remove the last ,\n - append(b"\n" + indent_chars + b"]") + append("\n" + indent_chars + "]") else: - append(b"[") + append("[") for elt in list_obj: serialize(elt, out, level + 1) - append(b",") + append(",") if list_obj: del out[-1] # remove the last , - append(b"]") + append("]") self.serialized_obj_ids.discard(id(list_obj)) dispatch[list] = ser_builtins_list @@ -429,9 +420,9 @@ def ser_builtins_dict(self, dict_obj, out, level): append = out.append serialize = self._serialize if self.indent and dict_obj: - indent_chars = b" " * level - indent_chars_inside = indent_chars + b" " - append(b"{\n") + indent_chars = " " * level + indent_chars_inside = indent_chars + " " + append("{\n") dict_items = dict_obj.items() try: sorted_items = sorted(dict_items) @@ -440,21 +431,21 @@ def ser_builtins_dict(self, dict_obj, out, level): for key, value in sorted_items: append(indent_chars_inside) serialize(key, out, level + 1) - append(b": ") + append(": ") serialize(value, out, level + 1) - append(b",\n") + append(",\n") del out[-1] # remove last ,\n - append(b"\n" + indent_chars + b"}") + append("\n" + indent_chars + "}") else: - append(b"{") + append("{") for key, value in dict_obj.items(): serialize(key, out, level + 1) - append(b":") + append(":") serialize(value, out, level + 1) - append(b",") + append(",") if dict_obj: del out[-1] # remove the last , - append(b"}") + append("}") self.serialized_obj_ids.discard(id(dict_obj)) dispatch[dict] = ser_builtins_dict @@ -467,9 +458,9 @@ def ser_builtins_set(self, set_obj, out, level): append = out.append serialize = self._serialize if self.indent and set_obj: - indent_chars = b" " * level - indent_chars_inside = indent_chars + b" " - append(b"{\n") + indent_chars = " " * level + indent_chars_inside = indent_chars + " " + append("{\n") try: sorted_elts = sorted(set_obj) except TypeError: # can occur when elements can't be ordered (Python 3.x) @@ -477,16 +468,16 @@ def ser_builtins_set(self, set_obj, out, level): for elt in sorted_elts: append(indent_chars_inside) serialize(elt, out, level + 1) - append(b",\n") + append(",\n") del out[-1] # remove the last ,\n - append(b"\n" + indent_chars + b"}") + append("\n" + indent_chars + "}") elif set_obj: - append(b"{") + append("{") for elt in set_obj: serialize(elt, out, level + 1) - append(b",") + append(",") del out[-1] # remove the last , - append(b"}") + append("}") else: # empty set literal doesn't exist unfortunately, replace with empty tuple self.ser_builtins_tuple((), out, level) @@ -498,33 +489,33 @@ def ser_builtins_frozenset(self, set_obj, out, level): def ser_decimal_Decimal(self, decimal_obj, out, level): # decimal is serialized as a string to avoid losing precision - self._serialize(str(decimal_obj), out, level) + out.append(repr(str(decimal_obj))) dispatch[decimal.Decimal] = ser_decimal_Decimal def ser_datetime_datetime(self, datetime_obj, out, level): - self._serialize(datetime_obj.isoformat(), out, level) + out.append(repr(datetime_obj.isoformat())) dispatch[datetime.datetime] = ser_datetime_datetime def ser_datetime_date(self, date_obj, out, level): - self._serialize(date_obj.isoformat(), out, level) + out.append(repr(date_obj.isoformat())) dispatch[datetime.date] = ser_datetime_date if os.name == "java" or sys.version_info < (2, 7): # jython bug http://bugs.jython.org/issue2010 def ser_datetime_timedelta(self, timedelta_obj, out, level): secs = ((timedelta_obj.days * 86400 + timedelta_obj.seconds) * 10 ** 6 + timedelta_obj.microseconds) / 10 ** 6 - self._serialize(secs, out, level) + out.append(repr(secs)) else: def ser_datetime_timedelta(self, timedelta_obj, out, level): secs = timedelta_obj.total_seconds() - self._serialize(secs, out, level) + out.append(repr(secs)) dispatch[datetime.timedelta] = ser_datetime_timedelta def ser_datetime_time(self, time_obj, out, level): - self._serialize(str(time_obj), out, level) + out.append(repr(str(time_obj))) dispatch[datetime.time] = ser_datetime_time def ser_uuid_UUID(self, uuid_obj, out, level): - self._serialize(str(uuid_obj), out, level) + out.append(repr(str(uuid_obj))) dispatch[uuid.UUID] = ser_uuid_UUID def ser_exception_class(self, exc_obj, out, level): diff --git a/tests/performance.py b/tests/performance.py index 9b86e95..a327a54 100644 --- a/tests/performance.py +++ b/tests/performance.py @@ -7,6 +7,8 @@ from timeit import default_timer as perf_timer import sys import datetime +import decimal +import uuid class Person(object): @@ -15,8 +17,9 @@ def __init__(self, name, age): self.age = age -data = { +guid = uuid.uuid4() +data = { "bytes": b"0123456789abcdefghijklmnopqrstuvwxyz" * 2000, "bytearray": bytearray(b"0123456789abcdefghijklmnopqrstuvwxyz") * 2000, "str": "\"0123456789\"\n'abcdefghijklmnopqrstuvwxyz'\t" * 2000, @@ -27,11 +30,13 @@ def __init__(self, name, age): "tuple": [(x * x, "tuple", (300, 400, (500, 600, (x * x, x * x)))) for x in range(200)], "list": [[x * x, "list", [300, 400, [500, 600, [x * x]]]] for x in range(200)], "set": set(x * x for x in range(1000)), - "dict": {i * i: {1000 + j: chr(j + 65) for j in range(5)} for i in range(100)}, + "dict": {str(i * i): {str(1000 + j): chr(j + 65) for j in range(5)} for i in range(100)}, "exception": [ZeroDivisionError("test exeception", x * x) for x in range(1000)], "class": [Person("harry", x * x) for x in range(1000)], "datetime": [datetime.datetime.now() for x in range(1000)], - "complex": [complex(x + x, x * x) for x in range(1000)] + "complex": [complex(x + x, x * x) for x in range(1000)], + "decimal": [decimal.Decimal("1122334455667788998877665544332211.9876543212345678987654321123456789") for x in range(1000)], + "uuid": [guid for x in range(1000)] } serializers = {} @@ -75,6 +80,7 @@ def run(): results = {} number = 10 repeat = 3 + serializers = {"serpent": (serpent.dumps, serpent.loads)} # XXX for ser in serializers: print("serializer:", ser) results[ser] = {"sizes": {}, "ser-times": {}, "deser-times": {}} From 7f6e95da4ca18f44281beb7dae0444352c3dd872 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sat, 18 Feb 2017 15:24:14 +0100 Subject: [PATCH 086/230] dotnet maxlevel also reduced to 500, version number. --- dotnet/Serpent.Test/CycleTest.cs | 2 +- dotnet/Serpent/Properties/AssemblyInfo.cs | 4 ++-- dotnet/Serpent/Serializer.cs | 2 +- dotnet/Serpent/Serpent.nuspec | 5 ++--- dotnet/Serpent/Version.cs | 2 +- java/src/main/java/net/razorvine/serpent/LibraryVersion.java | 2 +- java/src/main/java/net/razorvine/serpent/package-info.java | 2 +- 7 files changed, 9 insertions(+), 10 deletions(-) diff --git a/dotnet/Serpent.Test/CycleTest.cs b/dotnet/Serpent.Test/CycleTest.cs index baea64c..c02a1e5 100644 --- a/dotnet/Serpent.Test/CycleTest.cs +++ b/dotnet/Serpent.Test/CycleTest.cs @@ -100,7 +100,7 @@ public void testClassCycle() public void testMaxLevel() { Serializer ser = new Serializer(); - Assert.AreEqual(800, ser.MaximumLevel); + Assert.AreEqual(500, ser.MaximumLevel); Object[] array = new Object[] { "level1", diff --git a/dotnet/Serpent/Properties/AssemblyInfo.cs b/dotnet/Serpent/Properties/AssemblyInfo.cs index 6649bf4..ec8bb83 100644 --- a/dotnet/Serpent/Properties/AssemblyInfo.cs +++ b/dotnet/Serpent/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ decoded there (using ast.literal_eval()). It can ofcourse also deserialize // // You can specify all the values or you can use the default the Revision and // Build Numbers by using the '*' as shown below: -[assembly: AssemblyVersion("1.17.0.*")] -[assembly: AssemblyFileVersion("1.17.0.0")] +[assembly: AssemblyVersion("1.18.0.*")] +[assembly: AssemblyFileVersion("1.18.0.0")] diff --git a/dotnet/Serpent/Serializer.cs b/dotnet/Serpent/Serializer.cs index a6f49c0..7b9f61f 100644 --- a/dotnet/Serpent/Serializer.cs +++ b/dotnet/Serpent/Serializer.cs @@ -44,7 +44,7 @@ public class Serializer /// This limit has been set to avoid troublesome stack overflow errors. /// (If it is reached, an IllegalArgumentException is thrown instead with a clear message) /// - public int MaximumLevel = 800; // avoids stackoverflow errors + public int MaximumLevel = 500; // avoids stackoverflow errors private static IDictionary> classToDictRegistry = new Dictionary>(); diff --git a/dotnet/Serpent/Serpent.nuspec b/dotnet/Serpent/Serpent.nuspec index ec4948a..aa3c0f7 100644 --- a/dotnet/Serpent/Serpent.nuspec +++ b/dotnet/Serpent/Serpent.nuspec @@ -2,7 +2,7 @@ Razorvine.Serpent - 1.17.0.0 + 1.18.0.0 Razorvine.Serpent Irmen de Jong Irmen de Jong @@ -19,8 +19,7 @@ such a Python expression itself, back into the equivalent .NET datatypes. More info for the Python version is on Pypi: https://pypi.python.org/pypi/serpent Serpent Python literal expression serialization -CRITICAL FIX: rewrote serialization and parsing of strings containing chars above 255, this didn't work. Now mimics python's repr(str) form as closely as possible. -Added Serializer.MaximumLevel to avoid too deep recursion resulting in stack overflow errors. +Serializer.MaximumLevel decreased to 500. Copyright 2017 serialization python pyro diff --git a/dotnet/Serpent/Version.cs b/dotnet/Serpent/Version.cs index 2123caa..0d02f6a 100644 --- a/dotnet/Serpent/Version.cs +++ b/dotnet/Serpent/Version.cs @@ -10,6 +10,6 @@ namespace Razorvine.Serpent { public static class LibraryVersion { - public static string Version = "1.17"; + public static string Version = "1.18"; } } diff --git a/java/src/main/java/net/razorvine/serpent/LibraryVersion.java b/java/src/main/java/net/razorvine/serpent/LibraryVersion.java index a1b9a15..2f39949 100644 --- a/java/src/main/java/net/razorvine/serpent/LibraryVersion.java +++ b/java/src/main/java/net/razorvine/serpent/LibraryVersion.java @@ -8,5 +8,5 @@ package net.razorvine.serpent; public final class LibraryVersion { - public static final String VERSION = "1.17"; + public static final String VERSION = "1.18"; } diff --git a/java/src/main/java/net/razorvine/serpent/package-info.java b/java/src/main/java/net/razorvine/serpent/package-info.java index bd86c6e..bd22e82 100644 --- a/java/src/main/java/net/razorvine/serpent/package-info.java +++ b/java/src/main/java/net/razorvine/serpent/package-info.java @@ -3,7 +3,7 @@ * (a.k.a. Python's ast.literal_eval in Java) * Software license: "MIT software license". See http://opensource.org/licenses/MIT * @author Irmen de Jong (irmen@razorvine.net) - * @version 1.17 + * @version 1.18 */ package net.razorvine.serpent; From 8c5c172ad4da56b773b8e2f88fddf20c7f63561c Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sun, 19 Feb 2017 14:00:03 +0100 Subject: [PATCH 087/230] notes --- dotnet/Serpent/Serpent.nuspec | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dotnet/Serpent/Serpent.nuspec b/dotnet/Serpent/Serpent.nuspec index aa3c0f7..c2944c4 100644 --- a/dotnet/Serpent/Serpent.nuspec +++ b/dotnet/Serpent/Serpent.nuspec @@ -19,6 +19,8 @@ such a Python expression itself, back into the equivalent .NET datatypes. More info for the Python version is on Pypi: https://pypi.python.org/pypi/serpent Serpent Python literal expression serialization +CRITICAL FIX: rewrote serialization and parsing of strings containing chars above 255, this didn't work. Now mimics python's repr(str) form as closely as possible. +Added Serializer.MaximumLevel to avoid too deep recursion resulting in stack overflow errors. Serializer.MaximumLevel decreased to 500. Copyright 2017 From f0348693e8a59590e6be4af70e5944bacade1c0c Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sun, 19 Feb 2017 14:05:23 +0100 Subject: [PATCH 088/230] [maven-release-plugin] prepare release serpent-1.18 --- java/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 256273d..d4973b5 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -9,7 +9,7 @@ net.razorvine serpent - 1.18-SNAPSHOT + 1.18 jar serpent @@ -67,7 +67,7 @@ It is an alternative to JSON to provide easy data integration between Java and P https://github.com/irmen/Serpent scm:git:https://github.com/irmen/Serpent.git scm:git:https://github.com/irmen/Serpent.git - HEAD + serpent-1.18 Github From be8b4788f0f7a6e4e2b332d1808a391ff1d7cdfe Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sun, 19 Feb 2017 14:05:31 +0100 Subject: [PATCH 089/230] [maven-release-plugin] prepare for next development iteration --- java/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index d4973b5..ff5109a 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -9,7 +9,7 @@ net.razorvine serpent - 1.18 + 1.19-SNAPSHOT jar serpent @@ -67,7 +67,7 @@ It is an alternative to JSON to provide easy data integration between Java and P https://github.com/irmen/Serpent scm:git:https://github.com/irmen/Serpent.git scm:git:https://github.com/irmen/Serpent.git - serpent-1.18 + HEAD Github From 117d599ea16195acc1c7600226e8e7c72f82bd96 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Mon, 20 Feb 2017 21:28:33 +0100 Subject: [PATCH 090/230] jython workaround for bug2008 is not reliable so added a check, and improved unit test for this situation. --- README.md | 7 +++++++ serpent.py | 18 ++++++++++++++---- tests/test_serpent.py | 19 ++++++++++++++++++- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index e0a052f..8829b1e 100644 --- a/README.md +++ b/README.md @@ -83,3 +83,10 @@ Floats +inf and -inf are handled via a trick, Float 'nan' cannot be handled and is represented by the special value: ``{'__class__':'float','value':'nan'}`` We chose not to encode it as just the string 'NaN' because that could cause memory issues when used in multiplications. + +Jython's ast module cannot properly parse some literal reprs of unicode strings. +This is a known bug http://bugs.jython.org/issue2008 +It seems to work when your server is Python 2.x but safest is perhaps to make +sure your data to parse contains only ascii strings when dealing with Jython. +Serpent checks for possible problems and will raise an error if it finds one, +rather than continuing with string data that might be incorrect. \ No newline at end of file diff --git a/serpent.py b/serpent.py index d1a77f4..3f5979e 100644 --- a/serpent.py +++ b/serpent.py @@ -47,6 +47,13 @@ We chose not to encode it as just the string 'NaN' because that could cause memory issues when used in multiplications. +Jython's ast module cannot properly parse some literal reprs of unicode strings. +This is a known bug http://bugs.jython.org/issue2008 +It seems to work when your server is Python 2.x but safest is perhaps to make +sure your data to parse contains only ascii strings when dealing with Jython. +Serpent checks for possible problems and will raise an error if it finds one, +rather than continuing with string data that might be incorrect. + Copyright by Irmen de Jong (irmen@razorvine.net) Software license: "MIT software license". See http://opensource.org/licenses/MIT """ @@ -66,7 +73,7 @@ import array import math -__version__ = "1.18" +__version__ = "1.18.1" __all__ = ["dump", "dumps", "load", "loads", "register_class", "unregister_class", "tobytes"] can_use_set_literals = sys.version_info >= (3, 2) # check if we can use set literals @@ -89,13 +96,16 @@ def loads(serialized_bytes): serialized = serialized_bytes.decode("utf-8") if sys.version_info < (3, 0): # python 2.x: parse with unicode_literals (promotes all strings to unicode) + # note: this doesn't work on jython... see bug http://bugs.jython.org/issue2008 + # so we add a safety net, to avoid working with incorrectly processed unicode strings serialized = compile(serialized, "", mode="eval", flags=ast.PyCF_ONLY_AST | __future__.unicode_literals.compiler_flag) if os.name == "java": - # The ast module in Jython will not have parsed this correctly into unicode literals. - # So we have to patch up the ast tree ourselves and decode Str nodes to unicode manually. - # See http://bugs.jython.org/issue2008 for node in ast.walk(serialized): if isinstance(node, ast.Str): + if isinstance(node.s, str) and any(c for c in node.s if c > '\x7f'): + # In this case there is risk of incorrectly parsed unicode data. Play safe and crash. + raise ValueError("cannot properly parse unicode string with ast in Jython, see bug http://bugs.jython.org/issue2008" + " - use python 2.x server or convert strings to ascii yourself first") node.s = node.s.decode("unicode-escape") try: if os.name != "java" and sys.platform != "cli": diff --git a/tests/test_serpent.py b/tests/test_serpent.py index d901297..6dffd33 100644 --- a/tests/test_serpent.py +++ b/tests/test_serpent.py @@ -49,7 +49,7 @@ def test_deserialize(self): self.assertEqual(555, data) def test_deserialize_unichr(self): - unicodestring = "euro" + unichr(0x20ac) + unicodestring = u"euro\u20ac" encoded = repr(unicodestring).encode("utf-8") data = serpent.loads(encoded) self.assertEqual(unicodestring, data) @@ -84,6 +84,23 @@ def test_unicode_escapes(self): class TestBasics(unittest.TestCase): + + def test_py2_py3_unicode_repr(self): + data = u"hello\u20ac" + py2repr = b"# serpent utf-8 python2.6\n'hello\\u20ac'" + result = serpent.loads(py2repr) + self.assertEqual(data, result, "must understand python 2.x repr form of unicode string") + py3repr = b"# serpent utf-8 python3.2\n'hello\xe2\x82\xac'" + try: + result = serpent.loads(py3repr) # jython fails this test. + if os.name != "java": + self.assertEqual(data, result, "must understand python 3.x repr form of unicode string") + except ValueError as x: + if os.name == "java": + self.assertIn("issue2008", str(x)) + else: + self.fail("non-jython must parse it correctly") + def test_header(self): ser = serpent.dumps(None, set_literals=True) if sys.platform == "cli": From 343de8e79f8d3c792fad2d96cb3c083f938f3547 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sat, 25 Feb 2017 12:46:26 +0100 Subject: [PATCH 091/230] msgpack added to performance test --- tests/performance.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/performance.py b/tests/performance.py index a327a54..b917419 100644 --- a/tests/performance.py +++ b/tests/performance.py @@ -56,6 +56,11 @@ def __init__(self, name, age): serializers["serpent"] = (serpent.dumps, serpent.loads) import marshal serializers["marshal"] = (marshal.dumps, marshal.loads) +try: + import msgpack + serializers["msgpack"] = (lambda d: msgpack.packb(d, use_bin_type=True), lambda d: msgpack.unpackb(d, encoding="utf-8")) +except ImportError: + pass try: import xmlrpclib as xmlrpc except ImportError: @@ -80,7 +85,6 @@ def run(): results = {} number = 10 repeat = 3 - serializers = {"serpent": (serpent.dumps, serpent.loads)} # XXX for ser in serializers: print("serializer:", ser) results[ser] = {"sizes": {}, "ser-times": {}, "deser-times": {}} From db967b9dd509b458e5b543f836caa52e366b0c01 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Fri, 3 Mar 2017 20:43:47 +0100 Subject: [PATCH 092/230] can now serialize and deserialize from buffer/memoryview types directly --- serpent.py | 18 ++++++++++++++++-- tests/test_serpent.py | 15 +++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/serpent.py b/serpent.py index 3f5979e..b81e43c 100644 --- a/serpent.py +++ b/serpent.py @@ -72,8 +72,9 @@ import uuid import array import math +import codecs -__version__ = "1.18.1" +__version__ = "1.19" __all__ = ["dump", "dumps", "load", "loads", "register_class", "unregister_class", "tobytes"] can_use_set_literals = sys.version_info >= (3, 2) # check if we can use set literals @@ -93,7 +94,18 @@ def loads(serialized_bytes): """Deserialize bytes back to object tree. Uses ast.literal_eval (safe).""" if b'\x00' in serialized_bytes: raise ValueError("The serpent data contains 0-bytes so it cannot be parsed by ast.literal_eval. Has it been corrupted?") - serialized = serialized_bytes.decode("utf-8") + if os.name == "java": + if type(serialized_bytes) is memoryview: + serialized_bytes = serialized_bytes.tobytes() + elif type(serialized_bytes) is buffer: + serialized_bytes = serialized_bytes[:] + serialized = serialized_bytes.decode("utf-8") + elif sys.platform == "cli": + if type(serialized_bytes) is memoryview: + serialized_bytes = serialized_bytes.tobytes() + serialized = codecs.decode(serialized_bytes, "utf-8") + else: + serialized = codecs.decode(serialized_bytes, "utf-8") if sys.version_info < (3, 0): # python 2.x: parse with unicode_literals (promotes all strings to unicode) # note: this doesn't work on jython... see bug http://bugs.jython.org/issue2008 @@ -282,6 +294,8 @@ def serialize(self, obj): else: header += "python2.6\n" # don't change this even though we don't support 2.6 any longer, otherwise we can't read older serpent strings out = [header] + if os.name == "java" and type(obj) is buffer: + obj = bytearray(obj) try: if os.name != "java" and sys.platform != "cli": gc.disable() diff --git a/tests/test_serpent.py b/tests/test_serpent.py index 6dffd33..a6c607d 100644 --- a/tests/test_serpent.py +++ b/tests/test_serpent.py @@ -82,6 +82,17 @@ def test_unicode_escapes(self): v = serpent.loads(b"'\U00022001'") self.assertEqual(u"\U00022001", v) + def test_input_types(self): + bytes_input = b"'text'" + bytearray_input = bytearray(bytes_input) + memview_input = memoryview(bytes_input) + self.assertEqual("text", serpent.loads(bytes_input)) + self.assertEqual("text", serpent.loads(bytearray_input)) + self.assertEqual("text", serpent.loads(memview_input)) + if sys.version_info < (3, 0): + buffer_input = buffer(bytes_input) + self.assertEqual("text", serpent.loads(buffer_input)) + class TestBasics(unittest.TestCase): @@ -481,6 +492,10 @@ def test_bytes(self): ser = serpent.dumps(memoryview(b"abcdef")) data = serpent.loads(ser) self.assertEqual({'encoding': 'base64', 'data': 'YWJjZGVm'}, data) + if sys.version_info < (3, 0): + ser = serpent.dumps(buffer(b"abcdef")) + data = serpent.loads(ser) + self.assertEqual({'encoding': 'base64', 'data': 'YWJjZGVm'}, data) def test_exception(self): x = ZeroDivisionError("wrong") From 72fc6de33afcb7c78b84dc688baddb8f7b34bcfe Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Fri, 3 Mar 2017 20:46:57 +0100 Subject: [PATCH 093/230] also test pypy3 --- .travis.yml | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d9b8721..2da28e0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ python: - "3.5" - "3.6" - "pypy" + - "pypy3" # Use fast travis build infrastructure explicitly sudo: false diff --git a/tox.ini b/tox.ini index 5f41c91..76e5c85 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=py27,py33,py34,py35,py36,pypy +envlist=py27,py33,py34,py35,py36,pypy,pypy3 [testenv] deps=pytz From 9dfd91fa5fec8be2cbf948c804d2e8a5749ba391 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Fri, 3 Mar 2017 21:02:38 +0100 Subject: [PATCH 094/230] fix for certain pypy versions when decoding base64 bytes --- serpent.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/serpent.py b/serpent.py index b81e43c..fdbfd62 100644 --- a/serpent.py +++ b/serpent.py @@ -258,7 +258,10 @@ def tobytes(obj): if isinstance(obj, _bytes_types): return obj if isinstance(obj, dict) and "data" in obj and obj.get("encoding") == "base64": - return base64.b64decode(obj["data"]) + try: + return base64.b64decode(obj["data"]) + except TypeError: + return base64.b64decode(obj["data"].encode("ascii")) # needed for certain older versions of pypy raise TypeError("argument is neither bytes nor serpent base64 encoded bytes dict") From b092778fa3a2218aa331cc79fb49b216ec7648b0 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Fri, 3 Mar 2017 21:20:46 +0100 Subject: [PATCH 095/230] fixed test for \x00 bytes in serialized data --- serpent.py | 4 ++-- tests/test_serpent.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/serpent.py b/serpent.py index fdbfd62..776c4cf 100644 --- a/serpent.py +++ b/serpent.py @@ -92,8 +92,6 @@ def dump(obj, file, indent=False, set_literals=can_use_set_literals, module_in_c def loads(serialized_bytes): """Deserialize bytes back to object tree. Uses ast.literal_eval (safe).""" - if b'\x00' in serialized_bytes: - raise ValueError("The serpent data contains 0-bytes so it cannot be parsed by ast.literal_eval. Has it been corrupted?") if os.name == "java": if type(serialized_bytes) is memoryview: serialized_bytes = serialized_bytes.tobytes() @@ -106,6 +104,8 @@ def loads(serialized_bytes): serialized = codecs.decode(serialized_bytes, "utf-8") else: serialized = codecs.decode(serialized_bytes, "utf-8") + if '\x00' in serialized: + raise ValueError("The serpent data contains 0-bytes so it cannot be parsed by ast.literal_eval. Has it been corrupted?") if sys.version_info < (3, 0): # python 2.x: parse with unicode_literals (promotes all strings to unicode) # note: this doesn't work on jython... see bug http://bugs.jython.org/issue2008 diff --git a/tests/test_serpent.py b/tests/test_serpent.py index a6c607d..7998281 100644 --- a/tests/test_serpent.py +++ b/tests/test_serpent.py @@ -243,9 +243,19 @@ def test_nullbytesunicode(self): def test_detectNullByte(self): with self.assertRaises(ValueError) as ex: - serpent.loads(b"contains\x00nullbyte") + serpent.loads(b"'contains\x00nullbyte'") self.fail("must fail") self.assertTrue("0-bytes" in str(ex.exception)) + with self.assertRaises(ValueError) as ex: + serpent.loads(bytearray(b"'contains\x00nullbyte'")) + self.fail("must fail") + self.assertTrue("0-bytes" in str(ex.exception)) + with self.assertRaises(ValueError) as ex: + serpent.loads(memoryview(b"'contains\x00nullbyte'")) + self.fail("must fail") + self.assertTrue("0-bytes" in str(ex.exception)) + serpent.loads(bytearray(b"'contains no nullbyte'")) + serpent.loads(memoryview(b"'contains no nullbyte'")) @unittest.skipIf(os.name == "java", "jython can't parse unicode U's") def test_unicode_U(self): From d7dd8554bf035ceb515442415d97ce2777db2a35 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Mon, 17 Apr 2017 15:18:06 +0200 Subject: [PATCH 096/230] added anaconda badge --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8829b1e..01ce5c7 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ Serpent serialization library (Python/.NET/Java) [![Latest Version](https://img.shields.io/pypi/v/Serpent.svg)](https://pypi.python.org/pypi/Serpent/) [![Maven Central](https://img.shields.io/maven-central/v/net.razorvine/serpent.svg)](http://search.maven.org/#search|ga|1|g%3A%22net.razorvine%22%20AND%20a%3A%22serpent%22) [![NuGet](https://img.shields.io/nuget/v/Razorvine.Serpent.svg)](https://www.nuget.org/packages/Razorvine.Serpent/) +[![Anaconda-Server Badge](https://anaconda.org/conda-forge/serpent/badges/version.svg)](https://anaconda.org/conda-forge/serpent) Serpent provides ast.literal_eval() compatible object tree serialization. It serializes an object tree into bytes (utf-8 encoded string) that can be decoded and then @@ -89,4 +90,4 @@ This is a known bug http://bugs.jython.org/issue2008 It seems to work when your server is Python 2.x but safest is perhaps to make sure your data to parse contains only ascii strings when dealing with Jython. Serpent checks for possible problems and will raise an error if it finds one, -rather than continuing with string data that might be incorrect. \ No newline at end of file +rather than continuing with string data that might be incorrect. From a15905f8e9cac5938c4be190b1f6de0c3a80b5e8 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Thu, 8 Jun 2017 01:06:35 +0200 Subject: [PATCH 097/230] added check for hashable dict keys and set elements, to avoid serialized data that can't be deserialized later. --- serpent.py | 14 +++++++++++++- tests/test_serpent.py | 11 +++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/serpent.py b/serpent.py index 776c4cf..cb6a70b 100644 --- a/serpent.py +++ b/serpent.py @@ -72,9 +72,10 @@ import uuid import array import math +import numbers import codecs -__version__ = "1.19" +__version__ = "1.20" __all__ = ["dump", "dumps", "load", "loads", "register_class", "unregister_class", "tobytes"] can_use_set_literals = sys.version_info >= (3, 2) # check if we can use set literals @@ -440,6 +441,13 @@ def ser_builtins_list(self, list_obj, out, level): self.serialized_obj_ids.discard(id(list_obj)) dispatch[list] = ser_builtins_list + def _check_hashable_type(self, t): + if t not in (bool, bytes, str, tuple) and not issubclass(t, numbers.Number): + if sys.version_info < (3, 0) and t is unicode: + return + raise TypeError("one of the keys in a dict or set is not of a primitive hashable type: " + + str(type(t)) + ". Use simple types as keys or use a list or tuple as container.") + def ser_builtins_dict(self, dict_obj, out, level): if id(dict_obj) in self.serialized_obj_ids: raise ValueError("Circular reference detected (dict)") @@ -457,6 +465,7 @@ def ser_builtins_dict(self, dict_obj, out, level): sorted_items = dict_items for key, value in sorted_items: append(indent_chars_inside) + self._check_hashable_type(type(key)) serialize(key, out, level + 1) append(": ") serialize(value, out, level + 1) @@ -466,6 +475,7 @@ def ser_builtins_dict(self, dict_obj, out, level): else: append("{") for key, value in dict_obj.items(): + self._check_hashable_type(type(key)) serialize(key, out, level + 1) append(":") serialize(value, out, level + 1) @@ -494,6 +504,7 @@ def ser_builtins_set(self, set_obj, out, level): sorted_elts = set_obj for elt in sorted_elts: append(indent_chars_inside) + self._check_hashable_type(type(elt)) serialize(elt, out, level + 1) append(",\n") del out[-1] # remove the last ,\n @@ -501,6 +512,7 @@ def ser_builtins_set(self, set_obj, out, level): elif set_obj: append("{") for elt in set_obj: + self._check_hashable_type(type(elt)) serialize(elt, out, level + 1) append(",") del out[-1] # remove the last , diff --git a/tests/test_serpent.py b/tests/test_serpent.py index 7998281..7925358 100644 --- a/tests/test_serpent.py +++ b/tests/test_serpent.py @@ -592,6 +592,17 @@ def test_class2(self): data = serpent.loads(ser) self.assertEqual('pprint.PrettyPrinter', data["__class__"]) + def test_class_hashable_key_check(self): + import pprint + pp = pprint.PrettyPrinter(stream="dummy", width=42) + with self.assertRaises(TypeError) as x: + serpent.dumps({1: 1, 2: 1, 3: 1, pp: 1}) # can only serialize simple types as dict keys (hashable) + self.assertTrue("hashable type" in str(x.exception)) + if serpent.can_use_set_literals: + with self.assertRaises(TypeError) as x: + serpent.dumps({1, 2, 3, pp}) # can only serialize simple typles as set elements (hashable) + self.assertTrue("hashable type" in str(x.exception)) + def test_array(self): ser = serpent.dumps(array.array('u', unicode("unicode"))) data = strip_header(ser) From e3d57aa595b400710cb1263ce60cdc175d24e0f0 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Thu, 8 Jun 2017 01:14:03 +0200 Subject: [PATCH 098/230] added check for hashable dict keys and set elements, to avoid serialized data that can't be deserialized later. (fix errormsg) --- serpent.py | 2 +- tests/test_serpent.py | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/serpent.py b/serpent.py index cb6a70b..d21d72c 100644 --- a/serpent.py +++ b/serpent.py @@ -446,7 +446,7 @@ def _check_hashable_type(self, t): if sys.version_info < (3, 0) and t is unicode: return raise TypeError("one of the keys in a dict or set is not of a primitive hashable type: " - + str(type(t)) + ". Use simple types as keys or use a list or tuple as container.") + + str(t) + ". Use simple types as keys or use a list or tuple as container.") def ser_builtins_dict(self, dict_obj, out, level): if id(dict_obj) in self.serialized_obj_ids: diff --git a/tests/test_serpent.py b/tests/test_serpent.py index 7925358..4162345 100644 --- a/tests/test_serpent.py +++ b/tests/test_serpent.py @@ -595,13 +595,23 @@ def test_class2(self): def test_class_hashable_key_check(self): import pprint pp = pprint.PrettyPrinter(stream="dummy", width=42) + with self.assertRaises(TypeError) as x: + serpent.dumps({1: 1, 2: 1, 3: 1, strip_header: 1}) # can only serialize simple types as dict keys (hashable) + self.assertTrue("hashable type" in str(x.exception)) with self.assertRaises(TypeError) as x: serpent.dumps({1: 1, 2: 1, 3: 1, pp: 1}) # can only serialize simple types as dict keys (hashable) self.assertTrue("hashable type" in str(x.exception)) - if serpent.can_use_set_literals: - with self.assertRaises(TypeError) as x: - serpent.dumps({1, 2, 3, pp}) # can only serialize simple typles as set elements (hashable) - self.assertTrue("hashable type" in str(x.exception)) + + @unittest.skipIf(not serpent.can_use_set_literals, reason="no problem if serpent doesn't serializes set literals") + def test_class_hashable_set_element_check(self): + import pprint + pp = pprint.PrettyPrinter(stream="dummy", width=42) + with self.assertRaises(TypeError) as x: + serpent.dumps({1, 2, 3, strip_header}) # can only serialize simple typles as set elements (hashable) + self.assertTrue("hashable type" in str(x.exception)) + with self.assertRaises(TypeError) as x: + serpent.dumps({1, 2, 3, pp}) # can only serialize simple typles as set elements (hashable) + self.assertTrue("hashable type" in str(x.exception)) def test_array(self): ser = serpent.dumps(array.array('u', unicode("unicode"))) From 66a98e34c38ad74703a6757bda179fcb53991f94 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Thu, 8 Jun 2017 01:26:29 +0200 Subject: [PATCH 099/230] fix test make target --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f84da02..be89f0d 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ install: python setup.py install test: - cd tests && python -E test_serpent.py + cd tests && (PYTHONPATH=.. python -s test_serpent.py) check: flake8 --exclude .tox --ignore E501 From dfc7fcd10d4b215fdea9f3c0166f257edefc6243 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Thu, 8 Jun 2017 01:31:58 +0200 Subject: [PATCH 100/230] optimized make targets --- Makefile | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index be89f0d..260f663 100644 --- a/Makefile +++ b/Makefile @@ -1,19 +1,14 @@ -.PHONY: all sdist wheel install upload clean test check +.PHONY: all dist install upload clean test check all: @echo "targets include sdist, wheel, upload, install, clean, test" -sdist: - python setup.py sdist - @echo "Look in the dist/ directory" - -wheel: - python setup.py bdist_wheel +dist: + python setup.py sdist bdist_wheel @echo "Look in the dist/ directory" upload: - python setup.py sdist upload - python setup.py bdist_wheel upload + python setup.py sdist bdist_wheel upload install: python setup.py install From 5fb8cf1339867a80d4076b23ee2caa883271522a Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Thu, 8 Jun 2017 02:12:05 +0200 Subject: [PATCH 101/230] python 3.4's enums are also allowed as hashable type. --- serpent.py | 9 ++++++--- tests/test_serpent.py | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/serpent.py b/serpent.py index d21d72c..e2aee87 100644 --- a/serpent.py +++ b/serpent.py @@ -75,7 +75,7 @@ import numbers import codecs -__version__ = "1.20" +__version__ = "1.21" __all__ = ["dump", "dumps", "load", "loads", "register_class", "unregister_class", "tobytes"] can_use_set_literals = sys.version_info >= (3, 2) # check if we can use set literals @@ -443,9 +443,12 @@ def ser_builtins_list(self, list_obj, out, level): def _check_hashable_type(self, t): if t not in (bool, bytes, str, tuple) and not issubclass(t, numbers.Number): - if sys.version_info < (3, 0) and t is unicode: + if sys.version_info >= (3, 4) and issubclass(t, enum.Enum): return - raise TypeError("one of the keys in a dict or set is not of a primitive hashable type: " + elif sys.version_info < (3, 0) and t is unicode: + return + else: + raise TypeError("one of the keys in a dict or set is not of a primitive hashable type: " + str(t) + ". Use simple types as keys or use a list or tuple as container.") def ser_builtins_dict(self, dict_obj, out, level): diff --git a/tests/test_serpent.py b/tests/test_serpent.py index 4162345..7bd46a5 100644 --- a/tests/test_serpent.py +++ b/tests/test_serpent.py @@ -613,6 +613,20 @@ def test_class_hashable_set_element_check(self): serpent.dumps({1, 2, 3, pp}) # can only serialize simple typles as set elements (hashable) self.assertTrue("hashable type" in str(x.exception)) + @unittest.skipIf(sys.version_info < (3, 4), reason="python 3.4 introduced enums") + def test_enum_hashable(self): + import enum + class Color(enum.Enum): + RED = 1 + GREEN = 2 + BLUE = 3 + data = serpent.dumps({"abc", Color.RED, Color.GREEN, Color.BLUE}) + orig = serpent.loads(data) + self.assertEqual({"abc", 1, 2, 3}, orig) + data = serpent.dumps({"abc": 1, Color.RED: 1, Color.GREEN: 1, Color.BLUE: 1}) + orig = serpent.loads(data) + self.assertEqual({"abc": 1, 1: 1, 2: 1, 3: 1}, orig) + def test_array(self): ser = serpent.dumps(array.array('u', unicode("unicode"))) data = strip_header(ser) From 97266e834f67b1827ebef18486f4a62add7a33ea Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Thu, 8 Jun 2017 19:35:44 +0200 Subject: [PATCH 102/230] register_class should be order preserving. --- serpent.py | 42 ++++++++++++++++++++++++------------------ tests/test_serpent.py | 17 +++++++++++++++++ 2 files changed, 41 insertions(+), 18 deletions(-) diff --git a/serpent.py b/serpent.py index e2aee87..de8e119 100644 --- a/serpent.py +++ b/serpent.py @@ -75,7 +75,7 @@ import numbers import codecs -__version__ = "1.21" +__version__ = "1.22" __all__ = ["dump", "dumps", "load", "loads", "register_class", "unregister_class", "tobytes"] can_use_set_literals = sys.version_info >= (3, 2) # check if we can use set literals @@ -146,18 +146,23 @@ def _ser_DictView(obj, serializer, outputstream, indentlevel): serializer.ser_builtins_list(obj, outputstream, indentlevel) -_special_classes_registry = { - collections.KeysView: _ser_DictView, - collections.ValuesView: _ser_DictView, - collections.ItemsView: _ser_DictView -} -if sys.version_info >= (2, 7): - _special_classes_registry[collections.OrderedDict] = _ser_OrderedDict -if sys.version_info >= (3, 4): - import enum - def _ser_Enum(obj, serializer, outputstream, indentlevel): - serializer._serialize(obj.value, outputstream, indentlevel) - _special_classes_registry[enum.Enum] = _ser_Enum +_special_classes_registry = {} # XXXcollections.OrderedDict() + + +def _reset_special_classes_registry(): + _special_classes_registry.clear() + _special_classes_registry[collections.KeysView] = _ser_DictView + _special_classes_registry[collections.ValuesView] = _ser_DictView + _special_classes_registry[collections.ItemsView] = _ser_DictView + if sys.version_info >= (2, 7): + _special_classes_registry[collections.OrderedDict] = _ser_OrderedDict + if sys.version_info >= (3, 4): + import enum + def _ser_Enum(obj, serializer, outputstream, indentlevel): + serializer._serialize(obj.value, outputstream, indentlevel) + _special_classes_registry[enum.Enum] = _ser_Enum + +_reset_special_classes_registry() def unregister_class(clazz): @@ -443,13 +448,14 @@ def ser_builtins_list(self, list_obj, out, level): def _check_hashable_type(self, t): if t not in (bool, bytes, str, tuple) and not issubclass(t, numbers.Number): - if sys.version_info >= (3, 4) and issubclass(t, enum.Enum): - return + if sys.version_info >= (3, 4): + import enum + if issubclass(t, enum.Enum): + return elif sys.version_info < (3, 0) and t is unicode: return - else: - raise TypeError("one of the keys in a dict or set is not of a primitive hashable type: " - + str(t) + ". Use simple types as keys or use a list or tuple as container.") + raise TypeError("one of the keys in a dict or set is not of a primitive hashable type: " + + str(t) + ". Use simple types as keys or use a list or tuple as container.") def ser_builtins_dict(self, dict_obj, out, level): if id(dict_obj) in self.serialized_obj_ids: diff --git a/tests/test_serpent.py b/tests/test_serpent.py index 7bd46a5..4d1f03d 100644 --- a/tests/test_serpent.py +++ b/tests/test_serpent.py @@ -973,6 +973,23 @@ def custom_uuid_translate(obj, serp, stream, level): finally: serpent.unregister_class(uuid.UUID) + def testRegisterOrderPreserving(self): + serpent._reset_special_classes_registry() + serpent.register_class(BaseClass, lambda: None) + serpent.register_class(SubClass, lambda: None) + classes = list(serpent._special_classes_registry) + self.assertEqual(collections.KeysView, classes.pop(0)) + self.assertEqual(collections.ValuesView, classes.pop(0)) + self.assertEqual(collections.ItemsView, classes.pop(0)) + if sys.version_info >= (2, 7): + self.assertEqual(collections.OrderedDict, classes.pop(0)) + if sys.version_info >= (3, 4): + import enum + self.assertEqual(enum.Enum, classes.pop(0)) + self.assertEqual(BaseClass, classes.pop(0)) + self.assertEqual(SubClass, classes.pop(0)) + self.assertEqual(0, len(classes)) + class TestPyro4(unittest.TestCase): def testException(self): From 0dabb0b9737026b44c6cd35fd5474c5a2f239d69 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Thu, 8 Jun 2017 19:40:08 +0200 Subject: [PATCH 103/230] register_class should be order preserving. --- serpent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serpent.py b/serpent.py index de8e119..af8bd4b 100644 --- a/serpent.py +++ b/serpent.py @@ -146,7 +146,7 @@ def _ser_DictView(obj, serializer, outputstream, indentlevel): serializer.ser_builtins_list(obj, outputstream, indentlevel) -_special_classes_registry = {} # XXXcollections.OrderedDict() +_special_classes_registry = collections.OrderedDict() # must be insert-order preserving to make sure of proper precedence rules def _reset_special_classes_registry(): From 96c1ad1db7a90230211472c343b50474a0e0ff6d Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sun, 2 Jul 2017 22:53:14 +0200 Subject: [PATCH 104/230] tighter limit on max recursion level --- serpent.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/serpent.py b/serpent.py index af8bd4b..f5f69fd 100644 --- a/serpent.py +++ b/serpent.py @@ -75,7 +75,7 @@ import numbers import codecs -__version__ = "1.22" +__version__ = "1.23" __all__ = ["dump", "dumps", "load", "loads", "register_class", "unregister_class", "tobytes"] can_use_set_literals = sys.version_info >= (3, 2) # check if we can use set literals @@ -292,7 +292,7 @@ def __init__(self, indent=False, set_literals=can_use_set_literals, module_in_cl self.module_in_classname = module_in_classname self.serialized_obj_ids = set() self.special_classes_registry_copy = None - self.maximum_level = min(sys.getrecursionlimit() // 3.5, 1000) + self.maximum_level = min(sys.getrecursionlimit() // 5, 1000) def serialize(self, obj): """Serialize the object tree to bytes.""" @@ -320,7 +320,8 @@ def serialize(self, obj): def _serialize(self, obj, out, level): if level > self.maximum_level: - raise ValueError("Object graph nesting too deep. Increase serializer.maximum_level if you think you need more.") + raise ValueError("Object graph nesting too deep. Increase serializer.maximum_level if you think you need more, " + " but this may cause a RecursionError instead if Python's recursion limit doesn't allow it.") t = type(obj) if t in _translate_types: obj = _translate_types[t](obj) From 853b3d2ef3a3ad4164a71c93cd2f77a650c16341 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Mon, 3 Jul 2017 00:07:12 +0200 Subject: [PATCH 105/230] fix for pypy low recursion limit in test --- tests/test_serpent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_serpent.py b/tests/test_serpent.py index 4d1f03d..8355537 100644 --- a/tests/test_serpent.py +++ b/tests/test_serpent.py @@ -1054,7 +1054,7 @@ def testClassCycle(self): # noinspection PyUnreachableCode def testMaxLevel(self): ser = serpent.Serializer() - self.assertGreater(ser.maximum_level, 20) # 20 seems quite low, but Pypy appears to have a low default recursionlimit (100) + self.assertGreater(ser.maximum_level, 10) # old Pypy appears to have a very low default recursionlimit array=[] arr=array for level in range(min(sys.getrecursionlimit()+10, 2000)): From 088db9255a3f6edcee8b47da8f2973858b5f3fbf Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Mon, 3 Jul 2017 22:28:20 +0200 Subject: [PATCH 106/230] Java 9 compatibility fixes, requires Java 8 at a minimum. Now requires Java 8 (JDK 1.8) or newer to build and run because we can no longer use the JAXB classes anymore to format ISO dates and encode/decode base-64. Version bump to 1.23. --- java/.classpath | 2 +- java/.settings/org.eclipse.jdt.core.prefs | 6 +- java/pom.xml | 185 +- .../net/razorvine/serpent/ComplexNumber.java | 174 +- .../net/razorvine/serpent/LibraryVersion.java | 2 +- .../java/net/razorvine/serpent/Parser.java | 5 +- .../net/razorvine/serpent/Serializer.java | 1362 +++++++------- .../net/razorvine/serpent/package-info.java | 2 +- .../razorvine/serpent/test/ParserTest.java | 26 +- .../razorvine/serpent/test/SerializeTest.java | 1564 +++++++++-------- 10 files changed, 1681 insertions(+), 1647 deletions(-) diff --git a/java/.classpath b/java/.classpath index 30e0063..2ef81d0 100644 --- a/java/.classpath +++ b/java/.classpath @@ -18,6 +18,6 @@ - + diff --git a/java/.settings/org.eclipse.jdt.core.prefs b/java/.settings/org.eclipse.jdt.core.prefs index 6472c7c..529ef07 100644 --- a/java/.settings/org.eclipse.jdt.core.prefs +++ b/java/.settings/org.eclipse.jdt.core.prefs @@ -1,13 +1,13 @@ eclipse.preferences.version=1 org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=1.6 +org.eclipse.jdt.core.compiler.compliance=1.8 org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate org.eclipse.jdt.core.compiler.problem.assertIdentifier=error org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning -org.eclipse.jdt.core.compiler.source=1.6 +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/java/pom.xml b/java/pom.xml index ff5109a..bd72eb2 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -1,92 +1,93 @@ - - 4.0.0 - - - org.sonatype.oss - oss-parent - 9 - - - net.razorvine - serpent - 1.19-SNAPSHOT - jar - - serpent - https://github.com/irmen/Serpent - - - UTF-8 - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.3 - - 1.7 - 1.7 - - - - org.apache.maven.plugins - maven-gpg-plugin - 1.5 - - - org.apache.maven.plugins - maven-release-plugin - 2.5.2 - - - org.apache.maven.plugins - maven-javadoc-plugin - - -Xdoclint:none - - - - - - - - junit - junit - 4.12 - test - - - Serpent serializes an object tree into a Python ast.literal_eval() compatible literal expression. It is safe to send serpent data to other machines over the network for instance -(because only 'safe' literals are encoded). -There is also a deserializer or parse provided that turns such a literal expression back into the appropriate Java object tree. - -It is an alternative to JSON to provide easy data integration between Java and Python. Serpent is more expressive as JSON (it supports more data types). - - https://github.com/irmen/Serpent - scm:git:https://github.com/irmen/Serpent.git - scm:git:https://github.com/irmen/Serpent.git - HEAD - - - Github - https://github.com/irmen/Serpent/issues - - - - - irmen - Irmen de Jong - irmen@razorvine.net - https://github.com/irmen - - - - - - MIT License - http://www.opensource.org/licenses/mit-license.php - - - + + 4.0.0 + + + org.sonatype.oss + oss-parent + 9 + + + net.razorvine + serpent + 1.23-SNAPSHOT + jar + + serpent + https://github.com/irmen/Serpent + + + UTF-8 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + -Xlint:deprecation + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.5 + + + org.apache.maven.plugins + maven-release-plugin + 2.5.2 + + + org.apache.maven.plugins + maven-javadoc-plugin + + -Xdoclint:none + + + + + + + + junit + junit + 4.12 + test + + + Serpent serializes an object tree into a Python ast.literal_eval() compatible literal expression. It is safe to send serpent data to other machines over the network for instance +(because only 'safe' literals are encoded). +There is also a deserializer or parse provided that turns such a literal expression back into the appropriate Java object tree. + +It is an alternative to JSON to provide easy data integration between Java and Python. Serpent is more expressive as JSON (it supports more data types). + + https://github.com/irmen/Serpent + scm:git:https://github.com/irmen/Serpent.git + scm:git:https://github.com/irmen/Serpent.git + HEAD + + + Github + https://github.com/irmen/Serpent/issues + + + + + irmen + Irmen de Jong + irmen@razorvine.net + https://github.com/irmen + + + + + + MIT License + http://www.opensource.org/licenses/mit-license.php + + + diff --git a/java/src/main/java/net/razorvine/serpent/ComplexNumber.java b/java/src/main/java/net/razorvine/serpent/ComplexNumber.java index 62d5744..a931ef1 100644 --- a/java/src/main/java/net/razorvine/serpent/ComplexNumber.java +++ b/java/src/main/java/net/razorvine/serpent/ComplexNumber.java @@ -1,87 +1,87 @@ -/** - * Serpent, a Python literal expression serializer/deserializer - * (a.k.a. Python's ast.literal_eval in Java) - * Software license: "MIT software license". See http://opensource.org/licenses/MIT - * @author Irmen de Jong (irmen@razorvine.net) - */ - -package net.razorvine.serpent; - -import java.io.Serializable; - -/** - * A complex number. - */ -public class ComplexNumber implements Serializable { - - private static final long serialVersionUID = 5396759273405612137L; - - public double real; - public double imaginary; - - public ComplexNumber(double r, double i) { - real=r; - imaginary=i; - } - - @Override - public String toString() - { - StringBuilder sb=new StringBuilder(); - sb.append(real); - if(imaginary>0) - sb.append('+'); - return sb.append(imaginary).append('i').toString(); - } - - public double Magnitude() { - return Math.sqrt(real * real + imaginary * imaginary); - } - - public void add(ComplexNumber other) - { - real += other.real; - imaginary += other.imaginary; - } - - public void subtract(ComplexNumber other) - { - real -= other.real; - imaginary -= other.imaginary; - } - - public void multiply(ComplexNumber other) - { - double new_real = real * other.real - imaginary * other.imaginary; - double new_imaginary = real * other.imaginary + imaginary * other.real; - real = new_real; - imaginary = new_imaginary; - } - - public void divide(ComplexNumber other) - { - double new_real = (real * other.real + imaginary * other.imaginary) / (other.real * other.real + other.imaginary * other.imaginary); - double new_imaginary = (imaginary * other.real - real * other.imaginary) / (other.real * other.real + other.imaginary * other.imaginary); - real = new_real; - imaginary = new_imaginary; - } - - @Override - public boolean equals(Object obj) - { - if(!(obj instanceof ComplexNumber)) - { - return false; - } - ComplexNumber other = (ComplexNumber) obj; - return real==other.real && imaginary==other.imaginary; - } - - @Override - public int hashCode() - { - Double r = new Double(real); - Double i = new Double(imaginary); - return r.hashCode() ^ i.hashCode(); - } -} +/** + * Serpent, a Python literal expression serializer/deserializer + * (a.k.a. Python's ast.literal_eval in Java) + * Software license: "MIT software license". See http://opensource.org/licenses/MIT + * @author Irmen de Jong (irmen@razorvine.net) + */ + +package net.razorvine.serpent; + +import java.io.Serializable; + +/** + * A complex number. + */ +public class ComplexNumber implements Serializable { + + private static final long serialVersionUID = 5396759273405612137L; + + public double real; + public double imaginary; + + public ComplexNumber(double r, double i) { + real=r; + imaginary=i; + } + + @Override + public String toString() + { + StringBuilder sb=new StringBuilder(); + sb.append(real); + if(imaginary>0) + sb.append('+'); + return sb.append(imaginary).append('i').toString(); + } + + public double Magnitude() { + return Math.sqrt(real * real + imaginary * imaginary); + } + + public void add(ComplexNumber other) + { + real += other.real; + imaginary += other.imaginary; + } + + public void subtract(ComplexNumber other) + { + real -= other.real; + imaginary -= other.imaginary; + } + + public void multiply(ComplexNumber other) + { + double new_real = real * other.real - imaginary * other.imaginary; + double new_imaginary = real * other.imaginary + imaginary * other.real; + real = new_real; + imaginary = new_imaginary; + } + + public void divide(ComplexNumber other) + { + double new_real = (real * other.real + imaginary * other.imaginary) / (other.real * other.real + other.imaginary * other.imaginary); + double new_imaginary = (imaginary * other.real - real * other.imaginary) / (other.real * other.real + other.imaginary * other.imaginary); + real = new_real; + imaginary = new_imaginary; + } + + @Override + public boolean equals(Object obj) + { + if(!(obj instanceof ComplexNumber)) + { + return false; + } + ComplexNumber other = (ComplexNumber) obj; + return real==other.real && imaginary==other.imaginary; + } + + @Override + public int hashCode() + { + Double r = Double.valueOf(real); + Double i = Double.valueOf(imaginary); + return r.hashCode() ^ i.hashCode(); + } +} diff --git a/java/src/main/java/net/razorvine/serpent/LibraryVersion.java b/java/src/main/java/net/razorvine/serpent/LibraryVersion.java index 2f39949..c02bb33 100644 --- a/java/src/main/java/net/razorvine/serpent/LibraryVersion.java +++ b/java/src/main/java/net/razorvine/serpent/LibraryVersion.java @@ -8,5 +8,5 @@ package net.razorvine.serpent; public final class LibraryVersion { - public static final String VERSION = "1.18"; + public static final String VERSION = "1.23"; } diff --git a/java/src/main/java/net/razorvine/serpent/Parser.java b/java/src/main/java/net/razorvine/serpent/Parser.java index 662ffc8..6a54eb8 100644 --- a/java/src/main/java/net/razorvine/serpent/Parser.java +++ b/java/src/main/java/net/razorvine/serpent/Parser.java @@ -10,14 +10,13 @@ import java.io.UnsupportedEncodingException; import java.math.BigInteger; import java.util.ArrayList; +import java.util.Base64; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import javax.xml.bind.DatatypeConverter; - import net.razorvine.serpent.ast.*; @@ -618,7 +617,7 @@ public static byte[] toBytes(Object obj) { { throw new IllegalArgumentException("argument is neither bytearray nor serpent base64 encoded bytes dict"); } - return DatatypeConverter.parseBase64Binary(data); + return Base64.getDecoder().decode(data); } if(obj instanceof byte[]) { diff --git a/java/src/main/java/net/razorvine/serpent/Serializer.java b/java/src/main/java/net/razorvine/serpent/Serializer.java index cbb7ce6..d106ec5 100644 --- a/java/src/main/java/net/razorvine/serpent/Serializer.java +++ b/java/src/main/java/net/razorvine/serpent/Serializer.java @@ -1,670 +1,692 @@ -/** - * Serpent, a Python literal expression serializer/deserializer - * (a.k.a. Python's ast.literal_eval in Java) - * Software license: "MIT software license". See http://opensource.org/licenses/MIT - * @author Irmen de Jong (irmen@razorvine.net) - */ - -package net.razorvine.serpent; - -import java.io.IOException; -import java.io.Serializable; -import java.io.StringWriter; -import java.lang.reflect.Array; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.math.BigDecimal; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.*; -import java.util.Map.Entry; - -import javax.xml.bind.DatatypeConverter; - - -/** - * Serialize an object tree to a byte stream. - * It is not thread-safe: make sure you're not making changes to the object tree that is being serialized. - */ -public class Serializer -{ - /** - * The maximum nesting level of the object graphs that you want to serialize. - * This limit has been set to avoid troublesome stack overflow errors. - * (If it is reached, an IllegalArgumentException is thrown instead with a clear message) - */ - public int maximumLevel = 500; // to avoid stack overflow errors - - /** - * Indent the resulting serpent serialization text? - */ - public boolean indent = false; - - /** - * Use set literals? - */ - public boolean setliterals = true; - - /** - * Include package name in class name, for classes that are serialized to dicts? - */ - public boolean packageInClassName = false; - - private static Map, IClassSerializer> classToDictRegistry = new HashMap, IClassSerializer>(); - - /** - * Create a Serpent serializer with default options. - */ - public Serializer() - { - } - - /** - * Create a Serpent serializer with custom options. - * @param indent should the output be indented to make it more readable? - * @param setliterals should set literals be used (recommended if you use newer Python versions to parse this) - * @param packageInClassName should the package name be included with the class name for classes that are serialized to dict? - */ - public Serializer(boolean indent, boolean setliterals, boolean packageInClassName) - { - this.indent = indent; - this.setliterals = setliterals; - this.packageInClassName = packageInClassName; - } - - /** - * Register a custom class serializer, if you want to tweak the serialization of classes that Serpent doesn't know about yet. - */ - public static void registerClass(Class clazz, IClassSerializer converter) - { - classToDictRegistry.put(clazz, converter); - } - - /** - * Serialize an object graph to a serpent serialized form. - */ - public byte[] serialize(Object obj) - { - StringWriter sw = new StringWriter(); - - if(this.setliterals) - sw.write("# serpent utf-8 python3.2\n"); // set-literals require python 3.2+ to deserialize (ast.literal_eval limitation) - else - sw.write("# serpent utf-8 python2.6\n"); - serialize(obj, sw, 0); - - sw.flush(); - final String ser = sw.toString(); - try { - sw.close(); - return ser.getBytes("utf-8"); - } catch (IOException x) { - throw new IllegalArgumentException("error creating output bytes: "+x); - } - } - - protected void serialize(Object obj, StringWriter sw, int level) - { - if(level>maximumLevel) - throw new IllegalArgumentException("Object graph nesting too deep. Increase serializer.maximumLevel if you think you need more."); - - if(obj!=null && obj.getClass().getName().startsWith("org.python.")) - obj = convertJythonObject(obj); - - // null -> None - // hashtables/dictionaries -> dict - // hashset -> set - // array -> tuple - // byte arrays --> base64 - // any other collection --> list - // date//uuid/exception -> custom mapping - // random class --> public javabean properties to dictionary - // primitive types --> simple mapping - - Class type = obj==null? null : obj.getClass(); - Class componentType = type==null? null : type.getComponentType(); - - // primitive array? - if(componentType!=null) - { - // byte array? encode as base-64 - if(componentType==Byte.TYPE) - { - serialize_bytes((byte[])obj, sw, level); - return; - } - else - { - serialize_primitive_array(obj, sw, level); - } - return; - } - - if(obj==null) - { - sw.write("None"); - } - else if(obj instanceof String) - { - serialize_string((String)obj, sw, level); - } - else if(type.isPrimitive() || isBoxed(type)) - { - serialize_primitive(obj, sw, level); - } - else if(obj instanceof Enum) - { - serialize_string(obj.toString(), sw, level); - } - else if(obj instanceof BigDecimal) - { - serialize_bigdecimal((BigDecimal)obj, sw, level); - } - else if(obj instanceof Number) - { - serialize_primitive(obj, sw, level); - } - else if(obj instanceof Date) - { - serialize_date((Date)obj, sw, level); - } - else if(obj instanceof Calendar) - { - serialize_calendar((Calendar)obj, sw, level); - } - else if(obj instanceof UUID) - { - serialize_uuid((UUID)obj, sw, level); - } - else if(obj instanceof Set) - { - serialize_set((Set)obj, sw, level); - } - else if(obj instanceof Map) - { - serialize_dict((Map)obj, sw, level); - } - else if(obj instanceof Collection) - { - serialize_collection((Collection)obj, sw, level); - } - else if(obj instanceof ComplexNumber) - { - serialize_complex((ComplexNumber)obj, sw, level); - } - else if(obj instanceof Exception) - { - serialize_exception((Exception)obj, sw, level); - } - else if(obj instanceof Serializable) - { - serialize_class(obj, sw, level); - } - else - { - throw new IllegalArgumentException("cannot serialize object of type "+type); - } - } - - /** - * When used from Jython directly, it sometimes passes some Jython specific - * classes to the serializer (such as org.python.core.PyComplex for a complex number). - * Due to the way these are constructed, Serpent is greatly confused, and will often - * end up in an endless loop eventually crashing with too deep nesting. - * For the know cases, we convert them here to appropriate representations. - */ - protected Object convertJythonObject(Object obj) - { - final Class clazz = obj.getClass(); - final String classname = clazz.getName(); - - try - { - // use reflection because I don't want to have a compiler dependency on Jython. - if(classname.equals("org.python.core.PyTuple")) { - return clazz.getMethod("toArray").invoke(obj); - } - else if(classname.equals("org.python.core.PyComplex")) { - Object pyImag = clazz.getMethod("getImag").invoke(obj); - Object pyReal = clazz.getMethod("getReal").invoke(obj); - Double imag = (Double) pyImag.getClass().getMethod("getValue").invoke(pyImag); - Double real = (Double) pyReal.getClass().getMethod("getValue").invoke(pyReal); - return new ComplexNumber(real, imag); - } - else if(classname.equals("org.python.core.PyByteArray")) { - Object pyStr = clazz.getMethod("__str__").invoke(obj); - return pyStr.getClass().getMethod("toBytes").invoke(pyStr); - } - else if(classname.equals("org.python.core.PyMemoryView")) { - Object pyBytes = clazz.getMethod("tobytes").invoke(obj); - return pyBytes.getClass().getMethod("toBytes").invoke(pyBytes); - } - } catch (ReflectiveOperationException e) { - throw new IllegalArgumentException("cannot serialize Jython object of type "+clazz, e); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("cannot serialize Jython object of type "+clazz, e); - } catch (SecurityException e) { - throw new IllegalArgumentException("cannot serialize Jython object of type "+clazz, e); - } - - // instead of an endless nesting loop, report a proper exception - throw new IllegalArgumentException("cannot serialize Jython object of type "+obj.getClass()); - } - - protected void serialize_collection(Collection collection, StringWriter sw, int level) - { - // output a list - sw.write("["); - serialize_sequence_elements(collection, false, sw, level+1); - if(this.indent && collection.size()>0) - { - for(int i=0; i elts, boolean trailingComma, StringWriter sw, int level) - { - if(elts.size()==0) - return; - int count=0; - if(this.indent) - { - sw.write("\n"); - String innerindent = ""; - for(int i=0; i set, StringWriter sw, int level) - { - if(!this.setliterals) - { - // output a tuple instead of a set-literal - serialize_tuple(set, sw, level); - return; - } - - if(set.size()>0) - { - sw.write("{"); - Collection output = set; - if(this.indent) - { - // try to sort the set - Set outputset = set; - try { - outputset = new TreeSet(set); - } catch (ClassCastException x) { - // ignore unsortable elements - } - output = outputset; - } - serialize_sequence_elements(output, false, sw, level+1); - - if(this.indent) - { - for(int i=0; i items = new ArrayList(length); - for(int i=0; i items, StringWriter sw, int level) - { - sw.write("("); - serialize_sequence_elements(items, items.size()==1, sw, level+1); - if(this.indent && items.size()>0) - { - for(int i=0; i dict = new HashMap(); - dict.put("data", str); - dict.put("encoding", "base64"); - serialize_dict(dict, sw, level); - } - - protected void serialize_dict(Map dict, StringWriter sw,int level) - { - if(dict.size()==0) - { - sw.write("{}"); - return; - } - - int counter=0; - if(this.indent) - { - String innerindent = " "; - for(int i=0; i outputdict = dict; - try { - outputdict = new TreeMap(dict); - } catch (ClassCastException x) { - // ignore unsortable keys - } - - for(Map.Entry e: outputdict.entrySet()) - { - sw.write(innerindent); - serialize(e.getKey(), sw, level+1); - sw.write(": "); - serialize(e.getValue(), sw, level+1); - counter++; - if(counter e: dict.entrySet()) - { - serialize(e.getKey(), sw, level+1); - sw.write(":"); - serialize(e.getValue(), sw, level+1); - counter++; - if(counter=0) - sw.write("+"); - serialize_primitive(cplx.imaginary, sw, level); - sw.write("j)"); - } - - protected void serialize_uuid(UUID obj, StringWriter sw, int level) - { - serialize_string(obj.toString(), sw, level); - } - - protected void serialize_bigdecimal(BigDecimal decimal, StringWriter sw, int level) - { - serialize_string(decimal.toEngineeringString(), sw, level); - } - - private static final HashSet> boxedTypes; - static { - boxedTypes = new HashSet>(); - boxedTypes.add(Boolean.class); - boxedTypes.add(Character.class); - boxedTypes.add(Byte.class); - boxedTypes.add(Short.class); - boxedTypes.add(Integer.class); - boxedTypes.add(Long.class); - boxedTypes.add(Float.class); - boxedTypes.add(Double.class); - }; - - protected boolean isBoxed(Class type) - { - return boxedTypes.contains(type); - } - - protected void serialize_class(Object obj, StringWriter sw, int level) - { - Map map; - IClassSerializer converter=getCustomConverter(obj.getClass()); - if(null!=converter) - { - map = converter.convert(obj); - } - else - { - map=new HashMap(); - try { - // note: don't use the java.bean api, because that is not available on Android. - for(Method m: obj.getClass().getMethods()) { - int modifiers = m.getModifiers(); - if((modifiers & Modifier.PUBLIC)!=0 && (modifiers & Modifier.STATIC)==0) { - String methodname = m.getName(); - int prefixlen = 0; - if(methodname.equals("getClass")) continue; - if(methodname.startsWith("get")) prefixlen=3; - else if(methodname.startsWith("is")) prefixlen=2; - else continue; - Object value = m.invoke(obj); - String name = methodname.substring(prefixlen); - if(name.length()==1) { - name = name.toLowerCase(); - } else { - if(!Character.isUpperCase(name.charAt(1))) { - name = Character.toLowerCase(name.charAt(0)) + name.substring(1); - } - } - map.put(name, value); - } - } - if(this.packageInClassName) - map.put("__class__", obj.getClass().getName()); - else - map.put("__class__", obj.getClass().getSimpleName()); - } catch (IllegalAccessException e) { - throw new IllegalArgumentException("couldn't introspect javabean: "+e); - } catch (InvocationTargetException e) { - throw new IllegalArgumentException("couldn't introspect javabean: "+e); - } - } - serialize_dict(map, sw, level); - } - - protected IClassSerializer getCustomConverter(Class type) { - IClassSerializer converter = classToDictRegistry.get(type.getClass()); - if(converter!=null) { - return converter; // exact match - } - - // check if there's a custom pickler registered for an interface or abstract base class - // that this object implements or inherits from. - for(Entry, IClassSerializer> x: classToDictRegistry.entrySet()) { - if(x.getKey().isAssignableFrom(type)) { - return x.getValue(); - } - } - - return null; - } - - protected void serialize_primitive(Object obj, StringWriter sw, int level) - { - if(obj instanceof Boolean || obj.getClass()==Boolean.TYPE) - { - sw.write(obj.equals(Boolean.TRUE)? "True": "False"); - } - else if (obj instanceof Float || obj.getClass()==Float.TYPE) - { - Float f = (Float)obj; - serialize_primitive(f.doubleValue(), sw, level); - } - else if (obj instanceof Double || obj.getClass()==Double.TYPE) - { - Double d = (Double) obj; - if(d.isInfinite()) { - // output a literal expression that overflows the float and results in +/-INF - if(d>0.0) { - sw.write("1e30000"); - } else { - sw.write("-1e30000"); - } - } - else if(d.isNaN()) { - // there's no literal expression for a float NaN... - sw.write("{'__class__':'float','value':'nan'}"); - } else { - sw.write(d.toString()); - } - } - else - { - sw.write(obj.toString()); - } - } - - // the repr translation table for characters 0x00-0xff - private final static String[] repr_255; - static { - repr_255=new String[256]; - for(int c=0; c<32; ++c) { - repr_255[c] = String.format("\\x%02x",c); - } - for(char c=0x20; c<0x7f; ++c) { - repr_255[c] = String.valueOf(c); - } - for(int c=0x7f; c<=0xa0; ++c) { - repr_255[c] = String.format("\\x%02x", c); - } - for(char c=0xa1; c<=0xff; ++c) { - repr_255[c] = String.valueOf(c); - } - // odd ones out: - repr_255['\t'] = "\\t"; - repr_255['\n'] = "\\n"; - repr_255['\r'] = "\\r"; - repr_255['\\'] = "\\\\"; - repr_255[0xad] = "\\xad"; - } - - protected void serialize_string(String str, StringWriter sw, int level) - { - // create a 'repr' string representation following the same escaping rules as python 3.x repr() does. - StringBuilder b=new StringBuilder(str.length()*2); - boolean containsSingleQuote=false; - boolean containsQuote=false; - for(int i=0; i dict; - IClassSerializer converter=classToDictRegistry.get(ex.getClass()); - if(null!=converter) - { - dict = converter.convert(ex); - } - else - { - dict = new HashMap(); - if(this.packageInClassName) - dict.put("__class__", ex.getClass().getName()); - else - dict.put("__class__", ex.getClass().getSimpleName()); - dict.put("__exception__", true); - dict.put("args", new String[]{ex.getMessage()}); - dict.put("attributes", java.util.Collections.EMPTY_MAP); - } - serialize_dict(dict, sw, level); - } -} +/** + * Serpent, a Python literal expression serializer/deserializer + * (a.k.a. Python's ast.literal_eval in Java) + * Software license: "MIT software license". See http://opensource.org/licenses/MIT + * @author Irmen de Jong (irmen@razorvine.net) + */ + +package net.razorvine.serpent; + +import java.io.IOException; +import java.io.Serializable; +import java.io.StringWriter; +import java.lang.reflect.Array; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.math.BigDecimal; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.Base64; +import java.util.Map.Entry; + + +/** + * Serialize an object tree to a byte stream. + * It is not thread-safe: make sure you're not making changes to the object tree that is being serialized. + */ +public class Serializer +{ + /** + * The maximum nesting level of the object graphs that you want to serialize. + * This limit has been set to avoid troublesome stack overflow errors. + * (If it is reached, an IllegalArgumentException is thrown instead with a clear message) + */ + public int maximumLevel = 500; // to avoid stack overflow errors + + /** + * Indent the resulting serpent serialization text? + */ + public boolean indent = false; + + /** + * Use set literals? + */ + public boolean setliterals = true; + + /** + * Include package name in class name, for classes that are serialized to dicts? + */ + public boolean packageInClassName = false; + + private static Map, IClassSerializer> classToDictRegistry = new HashMap, IClassSerializer>(); + + /** + * Create a Serpent serializer with default options. + */ + public Serializer() + { + } + + /** + * Create a Serpent serializer with custom options. + * @param indent should the output be indented to make it more readable? + * @param setliterals should set literals be used (recommended if you use newer Python versions to parse this) + * @param packageInClassName should the package name be included with the class name for classes that are serialized to dict? + */ + public Serializer(boolean indent, boolean setliterals, boolean packageInClassName) + { + this.indent = indent; + this.setliterals = setliterals; + this.packageInClassName = packageInClassName; + } + + /** + * Register a custom class serializer, if you want to tweak the serialization of classes that Serpent doesn't know about yet. + */ + public static void registerClass(Class clazz, IClassSerializer converter) + { + classToDictRegistry.put(clazz, converter); + } + + /** + * Serialize an object graph to a serpent serialized form. + */ + public byte[] serialize(Object obj) + { + StringWriter sw = new StringWriter(); + + if(this.setliterals) + sw.write("# serpent utf-8 python3.2\n"); // set-literals require python 3.2+ to deserialize (ast.literal_eval limitation) + else + sw.write("# serpent utf-8 python2.6\n"); + serialize(obj, sw, 0); + + sw.flush(); + final String ser = sw.toString(); + try { + sw.close(); + return ser.getBytes("utf-8"); + } catch (IOException x) { + throw new IllegalArgumentException("error creating output bytes: "+x); + } + } + + protected void serialize(Object obj, StringWriter sw, int level) + { + if(level>maximumLevel) + throw new IllegalArgumentException("Object graph nesting too deep. Increase serializer.maximumLevel if you think you need more."); + + if(obj!=null && obj.getClass().getName().startsWith("org.python.")) + obj = convertJythonObject(obj); + + // null -> None + // hashtables/dictionaries -> dict + // hashset -> set + // array -> tuple + // byte arrays --> base64 + // any other collection --> list + // date//uuid/exception -> custom mapping + // random class --> public javabean properties to dictionary + // primitive types --> simple mapping + + Class type = obj==null? null : obj.getClass(); + Class componentType = type==null? null : type.getComponentType(); + + // primitive array? + if(componentType!=null) + { + // byte array? encode as base-64 + if(componentType==Byte.TYPE) + { + serialize_bytes((byte[])obj, sw, level); + return; + } + else + { + serialize_primitive_array(obj, sw, level); + } + return; + } + + if(obj==null) + { + sw.write("None"); + } + else if(obj instanceof String) + { + serialize_string((String)obj, sw, level); + } + else if(type.isPrimitive() || isBoxed(type)) + { + serialize_primitive(obj, sw, level); + } + else if(obj instanceof Enum) + { + serialize_string(obj.toString(), sw, level); + } + else if(obj instanceof BigDecimal) + { + serialize_bigdecimal((BigDecimal)obj, sw, level); + } + else if(obj instanceof Number) + { + serialize_primitive(obj, sw, level); + } + else if(obj instanceof Date) + { + serialize_date((Date)obj, sw, level); + } + else if(obj instanceof Calendar) + { + serialize_calendar((Calendar)obj, sw, level); + } + else if(obj instanceof UUID) + { + serialize_uuid((UUID)obj, sw, level); + } + else if(obj instanceof Set) + { + serialize_set((Set)obj, sw, level); + } + else if(obj instanceof Map) + { + serialize_dict((Map)obj, sw, level); + } + else if(obj instanceof Collection) + { + serialize_collection((Collection)obj, sw, level); + } + else if(obj instanceof ComplexNumber) + { + serialize_complex((ComplexNumber)obj, sw, level); + } + else if(obj instanceof Exception) + { + serialize_exception((Exception)obj, sw, level); + } + else if(obj instanceof Serializable) + { + serialize_class(obj, sw, level); + } + else + { + throw new IllegalArgumentException("cannot serialize object of type "+type); + } + } + + /** + * When used from Jython directly, it sometimes passes some Jython specific + * classes to the serializer (such as org.python.core.PyComplex for a complex number). + * Due to the way these are constructed, Serpent is greatly confused, and will often + * end up in an endless loop eventually crashing with too deep nesting. + * For the know cases, we convert them here to appropriate representations. + */ + protected Object convertJythonObject(Object obj) + { + final Class clazz = obj.getClass(); + final String classname = clazz.getName(); + + try + { + // use reflection because I don't want to have a compiler dependency on Jython. + if(classname.equals("org.python.core.PyTuple")) { + return clazz.getMethod("toArray").invoke(obj); + } + else if(classname.equals("org.python.core.PyComplex")) { + Object pyImag = clazz.getMethod("getImag").invoke(obj); + Object pyReal = clazz.getMethod("getReal").invoke(obj); + Double imag = (Double) pyImag.getClass().getMethod("getValue").invoke(pyImag); + Double real = (Double) pyReal.getClass().getMethod("getValue").invoke(pyReal); + return new ComplexNumber(real, imag); + } + else if(classname.equals("org.python.core.PyByteArray")) { + Object pyStr = clazz.getMethod("__str__").invoke(obj); + return pyStr.getClass().getMethod("toBytes").invoke(pyStr); + } + else if(classname.equals("org.python.core.PyMemoryView")) { + Object pyBytes = clazz.getMethod("tobytes").invoke(obj); + return pyBytes.getClass().getMethod("toBytes").invoke(pyBytes); + } + } catch (ReflectiveOperationException e) { + throw new IllegalArgumentException("cannot serialize Jython object of type "+clazz, e); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("cannot serialize Jython object of type "+clazz, e); + } catch (SecurityException e) { + throw new IllegalArgumentException("cannot serialize Jython object of type "+clazz, e); + } + + // instead of an endless nesting loop, report a proper exception + throw new IllegalArgumentException("cannot serialize Jython object of type "+obj.getClass()); + } + + protected void serialize_collection(Collection collection, StringWriter sw, int level) + { + // output a list + sw.write("["); + serialize_sequence_elements(collection, false, sw, level+1); + if(this.indent && collection.size()>0) + { + for(int i=0; i elts, boolean trailingComma, StringWriter sw, int level) + { + if(elts.size()==0) + return; + int count=0; + if(this.indent) + { + sw.write("\n"); + String innerindent = ""; + for(int i=0; i set, StringWriter sw, int level) + { + if(!this.setliterals) + { + // output a tuple instead of a set-literal + serialize_tuple(set, sw, level); + return; + } + + if(set.size()>0) + { + sw.write("{"); + Collection output = set; + if(this.indent) + { + // try to sort the set + Set outputset = set; + try { + outputset = new TreeSet(set); + } catch (ClassCastException x) { + // ignore unsortable elements + } + output = outputset; + } + serialize_sequence_elements(output, false, sw, level+1); + + if(this.indent) + { + for(int i=0; i items = new ArrayList(length); + for(int i=0; i items, StringWriter sw, int level) + { + sw.write("("); + serialize_sequence_elements(items, items.size()==1, sw, level+1); + if(this.indent && items.size()>0) + { + for(int i=0; i dict = new HashMap(); + dict.put("data", str); + dict.put("encoding", "base64"); + serialize_dict(dict, sw, level); + } + + protected void serialize_dict(Map dict, StringWriter sw,int level) + { + if(dict.size()==0) + { + sw.write("{}"); + return; + } + + int counter=0; + if(this.indent) + { + String innerindent = " "; + for(int i=0; i outputdict = dict; + try { + outputdict = new TreeMap(dict); + } catch (ClassCastException x) { + // ignore unsortable keys + } + + for(Map.Entry e: outputdict.entrySet()) + { + sw.write(innerindent); + serialize(e.getKey(), sw, level+1); + sw.write(": "); + serialize(e.getValue(), sw, level+1); + counter++; + if(counter e: dict.entrySet()) + { + serialize(e.getKey(), sw, level+1); + sw.write(":"); + serialize(e.getValue(), sw, level+1); + counter++; + if(counter0) { + // we have millis + df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS"+tzformat); + } else { + // no millis + df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"+tzformat); + } + df.setTimeZone(cal.getTimeZone()); + serialize_string(df.format(cal.getTime()), sw, level); + } + + protected void serialize_date(Date date, StringWriter sw, int level) + { + DateFormat df; + + if((date.getTime() % 1000) != 0) { + // we have millis + df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + } else { + // no millis + df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + } + df.setTimeZone(TimeZone.getTimeZone("UTC")); + serialize_string(df.format(date), sw, level); + } + + protected void serialize_complex(ComplexNumber cplx, StringWriter sw, int level) + { + sw.write("("); + serialize_primitive(cplx.real, sw, level); + if(cplx.imaginary>=0) + sw.write("+"); + serialize_primitive(cplx.imaginary, sw, level); + sw.write("j)"); + } + + protected void serialize_uuid(UUID obj, StringWriter sw, int level) + { + serialize_string(obj.toString(), sw, level); + } + + protected void serialize_bigdecimal(BigDecimal decimal, StringWriter sw, int level) + { + serialize_string(decimal.toEngineeringString(), sw, level); + } + + private static final HashSet> boxedTypes; + static { + boxedTypes = new HashSet>(); + boxedTypes.add(Boolean.class); + boxedTypes.add(Character.class); + boxedTypes.add(Byte.class); + boxedTypes.add(Short.class); + boxedTypes.add(Integer.class); + boxedTypes.add(Long.class); + boxedTypes.add(Float.class); + boxedTypes.add(Double.class); + }; + + protected boolean isBoxed(Class type) + { + return boxedTypes.contains(type); + } + + protected void serialize_class(Object obj, StringWriter sw, int level) + { + Map map; + IClassSerializer converter=getCustomConverter(obj.getClass()); + if(null!=converter) + { + map = converter.convert(obj); + } + else + { + map=new HashMap(); + try { + // note: don't use the java.bean api, because that is not available on Android. + for(Method m: obj.getClass().getMethods()) { + int modifiers = m.getModifiers(); + if((modifiers & Modifier.PUBLIC)!=0 && (modifiers & Modifier.STATIC)==0) { + String methodname = m.getName(); + int prefixlen = 0; + if(methodname.equals("getClass")) continue; + if(methodname.startsWith("get")) prefixlen=3; + else if(methodname.startsWith("is")) prefixlen=2; + else continue; + Object value = m.invoke(obj); + String name = methodname.substring(prefixlen); + if(name.length()==1) { + name = name.toLowerCase(); + } else { + if(!Character.isUpperCase(name.charAt(1))) { + name = Character.toLowerCase(name.charAt(0)) + name.substring(1); + } + } + map.put(name, value); + } + } + if(this.packageInClassName) + map.put("__class__", obj.getClass().getName()); + else + map.put("__class__", obj.getClass().getSimpleName()); + } catch (IllegalAccessException e) { + throw new IllegalArgumentException("couldn't introspect javabean: "+e); + } catch (InvocationTargetException e) { + throw new IllegalArgumentException("couldn't introspect javabean: "+e); + } + } + serialize_dict(map, sw, level); + } + + protected IClassSerializer getCustomConverter(Class type) { + IClassSerializer converter = classToDictRegistry.get(type.getClass()); + if(converter!=null) { + return converter; // exact match + } + + // check if there's a custom pickler registered for an interface or abstract base class + // that this object implements or inherits from. + for(Entry, IClassSerializer> x: classToDictRegistry.entrySet()) { + if(x.getKey().isAssignableFrom(type)) { + return x.getValue(); + } + } + + return null; + } + + protected void serialize_primitive(Object obj, StringWriter sw, int level) + { + if(obj instanceof Boolean || obj.getClass()==Boolean.TYPE) + { + sw.write(obj.equals(Boolean.TRUE)? "True": "False"); + } + else if (obj instanceof Float || obj.getClass()==Float.TYPE) + { + Float f = (Float)obj; + serialize_primitive(f.doubleValue(), sw, level); + } + else if (obj instanceof Double || obj.getClass()==Double.TYPE) + { + Double d = (Double) obj; + if(d.isInfinite()) { + // output a literal expression that overflows the float and results in +/-INF + if(d>0.0) { + sw.write("1e30000"); + } else { + sw.write("-1e30000"); + } + } + else if(d.isNaN()) { + // there's no literal expression for a float NaN... + sw.write("{'__class__':'float','value':'nan'}"); + } else { + sw.write(d.toString()); + } + } + else + { + sw.write(obj.toString()); + } + } + + // the repr translation table for characters 0x00-0xff + private final static String[] repr_255; + static { + repr_255=new String[256]; + for(int c=0; c<32; ++c) { + repr_255[c] = String.format("\\x%02x",c); + } + for(char c=0x20; c<0x7f; ++c) { + repr_255[c] = String.valueOf(c); + } + for(int c=0x7f; c<=0xa0; ++c) { + repr_255[c] = String.format("\\x%02x", c); + } + for(char c=0xa1; c<=0xff; ++c) { + repr_255[c] = String.valueOf(c); + } + // odd ones out: + repr_255['\t'] = "\\t"; + repr_255['\n'] = "\\n"; + repr_255['\r'] = "\\r"; + repr_255['\\'] = "\\\\"; + repr_255[0xad] = "\\xad"; + } + + protected void serialize_string(String str, StringWriter sw, int level) + { + // create a 'repr' string representation following the same escaping rules as python 3.x repr() does. + StringBuilder b=new StringBuilder(str.length()*2); + boolean containsSingleQuote=false; + boolean containsQuote=false; + for(int i=0; i dict; + IClassSerializer converter=classToDictRegistry.get(ex.getClass()); + if(null!=converter) + { + dict = converter.convert(ex); + } + else + { + dict = new HashMap(); + if(this.packageInClassName) + dict.put("__class__", ex.getClass().getName()); + else + dict.put("__class__", ex.getClass().getSimpleName()); + dict.put("__exception__", true); + dict.put("args", new String[]{ex.getMessage()}); + dict.put("attributes", java.util.Collections.EMPTY_MAP); + } + serialize_dict(dict, sw, level); + } +} diff --git a/java/src/main/java/net/razorvine/serpent/package-info.java b/java/src/main/java/net/razorvine/serpent/package-info.java index bd22e82..d5a661b 100644 --- a/java/src/main/java/net/razorvine/serpent/package-info.java +++ b/java/src/main/java/net/razorvine/serpent/package-info.java @@ -3,7 +3,7 @@ * (a.k.a. Python's ast.literal_eval in Java) * Software license: "MIT software license". See http://opensource.org/licenses/MIT * @author Irmen de Jong (irmen@razorvine.net) - * @version 1.18 + * @version 1.23 */ package net.razorvine.serpent; diff --git a/java/src/test/java/net/razorvine/serpent/test/ParserTest.java b/java/src/test/java/net/razorvine/serpent/test/ParserTest.java index 92a05fa..d4b644f 100644 --- a/java/src/test/java/net/razorvine/serpent/test/ParserTest.java +++ b/java/src/test/java/net/razorvine/serpent/test/ParserTest.java @@ -136,23 +136,23 @@ public void TestFloatPrecision() Serializer serpent = new Serializer(); byte[] ser = serpent.serialize(1.2345678987654321); DoubleNode dv = (DoubleNode) p.parse(ser).root; - assertEquals(new Double(1.2345678987654321), dv.value); + assertEquals(Double.valueOf(1.2345678987654321), dv.value); ser = serpent.serialize(5555.12345678987656); dv = (DoubleNode) p.parse(ser).root; - assertEquals(new Double(5555.12345678987656), dv.value); + assertEquals(Double.valueOf(5555.12345678987656), dv.value); ser = serpent.serialize(98765432123456.12345678987656); dv = (DoubleNode) p.parse(ser).root; - assertEquals(new Double(98765432123456.12345678987656), dv.value); + assertEquals(Double.valueOf(98765432123456.12345678987656), dv.value); ser = serpent.serialize(98765432123456.12345678987656e+44); dv = (DoubleNode) p.parse(ser).root; - assertEquals(new Double(98765432123456.12345678987656e+44), dv.value); + assertEquals(Double.valueOf(98765432123456.12345678987656e+44), dv.value); ser = serpent.serialize(-98765432123456.12345678987656e-44); dv = (DoubleNode) p.parse(ser).root; - assertEquals(new Double(-98765432123456.12345678987656e-44), dv.value); + assertEquals(Double.valueOf(-98765432123456.12345678987656e-44), dv.value); } @Test @@ -485,17 +485,17 @@ public void TestComplexPrecision() { Parser p = new Parser(); ComplexNumberNode cv = (ComplexNumberNode)p.parse("(98765432123456.12345678987656+665544332211.9998877665544j)").root; - assertEquals(new Double(98765432123456.12345678987656), cv.real, 0); - assertEquals(new Double(665544332211.9998877665544), cv.imaginary, 0); + assertEquals(Double.valueOf(98765432123456.12345678987656), cv.real, 0); + assertEquals(Double.valueOf(665544332211.9998877665544), cv.imaginary, 0); cv = (ComplexNumberNode)p.parse("(98765432123456.12345678987656-665544332211.9998877665544j)").root; - assertEquals(new Double(98765432123456.12345678987656), cv.real, 0); - assertEquals(new Double(-665544332211.9998877665544), cv.imaginary, 0); + assertEquals(Double.valueOf(98765432123456.12345678987656), cv.real, 0); + assertEquals(Double.valueOf(-665544332211.9998877665544), cv.imaginary, 0); cv = (ComplexNumberNode)p.parse("(98765432123456.12345678987656e+33+665544332211.9998877665544e+44j)").root; - assertEquals(new Double(98765432123456.12345678987656e+33), cv.real, 0); - assertEquals(new Double(665544332211.9998877665544e+44), cv.imaginary, 0); + assertEquals(Double.valueOf(98765432123456.12345678987656e+33), cv.real, 0); + assertEquals(Double.valueOf(665544332211.9998877665544e+44), cv.imaginary, 0); cv = (ComplexNumberNode)p.parse("(-98765432123456.12345678987656e+33-665544332211.9998877665544e+44j)").root; - assertEquals(new Double(-98765432123456.12345678987656e+33), cv.real, 0); - assertEquals(new Double(-665544332211.9998877665544e+44), cv.imaginary, 0); + assertEquals(Double.valueOf(-98765432123456.12345678987656e+33), cv.real, 0); + assertEquals(Double.valueOf(-665544332211.9998877665544e+44), cv.imaginary, 0); } @Test diff --git a/java/src/test/java/net/razorvine/serpent/test/SerializeTest.java b/java/src/test/java/net/razorvine/serpent/test/SerializeTest.java index 5f0043d..68d2ccd 100644 --- a/java/src/test/java/net/razorvine/serpent/test/SerializeTest.java +++ b/java/src/test/java/net/razorvine/serpent/test/SerializeTest.java @@ -1,776 +1,788 @@ -/** - * Serpent, a Python literal expression serializer/deserializer - * (a.k.a. Python's ast.literal_eval in Java) - * Software license: "MIT software license". See http://opensource.org/licenses/MIT - * @author Irmen de Jong (irmen@razorvine.net) - */ - -package net.razorvine.serpent.test; - -import static org.junit.Assert.*; - -import java.io.Serializable; -import java.io.UnsupportedEncodingException; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.util.*; - -import net.razorvine.serpent.ComplexNumber; -import net.razorvine.serpent.IClassSerializer; -import net.razorvine.serpent.Parser; -import net.razorvine.serpent.Serializer; - -import org.junit.Test; - -public class SerializeTest { - - public byte[] strip_header(byte[] data) - { - int start; - for(start=0; start=data.length) - { - throw new IllegalArgumentException("need header in string"); - } - start++; - byte[] result = new byte[data.length-start]; - System.arraycopy(data, start, result, 0, data.length-start); - return result; - } - - public byte[] B(String s) - { - try { - return s.getBytes("utf-8"); - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - return null; - } - } - - public String S(byte[] b) - { - try { - return new String(b, "utf-8"); - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - return null; - } - } - - - @Test - public void testHeader() - { - Serializer ser = new Serializer(); - byte[] data = ser.serialize(null); - assertEquals(35, data[0]); - String strdata = S(data); - assertEquals("# serpent utf-8 python3.2", strdata.split("\n")[0]); - - ser.setliterals=false; - data = ser.serialize(null); - strdata = S(data); - assertEquals("# serpent utf-8 python2.6", strdata.split("\n")[0]); - - data = B("# header\nfirst-line"); - data = strip_header(data); - assertEquals("first-line", S(data)); - } - - - @Test - public void testException() - { - Serializer.registerClass(IllegalArgumentException.class, null); - Exception x = new IllegalArgumentException("errormessage"); - Serializer serpent = new Serializer(true, true, false); - byte[] ser = strip_header(serpent.serialize(x)); - assertEquals("{\n '__class__': 'IllegalArgumentException',\n '__exception__': True,\n 'args': (\n 'errormessage',\n ),\n 'attributes': {}\n}", S(ser)); - } - - @Test - public void testExceptionPackage() - { - Serializer.registerClass(IllegalArgumentException.class, null); - Exception x = new IllegalArgumentException("errormessage"); - Serializer serpent = new Serializer(true, true, true); - byte[] ser = strip_header(serpent.serialize(x)); - assertEquals("{\n '__class__': 'java.lang.IllegalArgumentException',\n '__exception__': True,\n 'args': (\n 'errormessage',\n ),\n 'attributes': {}\n}", S(ser)); - } - - @Test - public void testStuff() - { - Serializer ser=new Serializer(); - byte[] result = ser.serialize("blerp"); - result=strip_header(result); - assertEquals("'blerp'", S(result)); - result = ser.serialize("\\"); - result=strip_header(result); - assertEquals("'\\\\'", S(result)); - result = ser.serialize(UUID.fromString("f1f8d00e-49a5-4662-ac1d-d5f0426ed293")); - result=strip_header(result); - assertEquals("'f1f8d00e-49a5-4662-ac1d-d5f0426ed293'", S(result)); - result = ser.serialize(new BigDecimal("123456789.987654321987654321987654321987654321")); - result=strip_header(result); - assertEquals("'123456789.987654321987654321987654321987654321'", S(result)); - } - - - @Test - public void testNull() - { - Serializer ser = new Serializer(); - byte[] data = ser.serialize(null); - data=strip_header(data); - assertEquals("None", S(data)); - } - - - @Test - public void testStrings() - { - Serializer serpent = new Serializer(); - byte[] ser = serpent.serialize("hello"); - byte[] data = strip_header(ser); - assertEquals("'hello'", S(data)); - ser = serpent.serialize("quotes'\""); - data = strip_header(ser); - assertEquals("'quotes\\'\"'", S(data)); - ser = serpent.serialize("quotes2'"); - data = strip_header(ser); - assertEquals("\"quotes2'\"", S(data)); - } - - @Test - public void testUnicodeEscapes() - { - Serializer serpent=new Serializer(); - - // regular escaped chars first - byte[] ser = serpent.serialize("\b\r\n\f\t \\"); - byte[] data = strip_header(ser); - // '\\x08\\r\\n\\x0c\\t \\\\' - assertArrayEquals(new byte[] {39, - 92, 120, 48, 56, - 92, 114, - 92, 110, - 92, 120, 48, 99, - 92, 116, - 32, - 92, 92, - 39}, data); - - // simple cases (chars < 0x80) - ser = serpent.serialize("\u0000\u0001\u001f\u007f"); - data = strip_header(ser); - // '\\x00\\x01\\x1f\\x7f' - assertArrayEquals(new byte[] {39, - 92, 120, 48, 48, - 92, 120, 48, 49, - 92, 120, 49, 102, - 92, 120, 55, 102, - 39 }, data); - - // chars 0x80 .. 0xff - ser = serpent.serialize("\u0080\u0081\u00ff"); - data = strip_header(ser); - // '\\x80\\x81\xc3\xbf' (has some utf-8 encoded chars in it) - assertArrayEquals(new byte[] {39, - 92, 120, 56, 48, - 92, 120, 56, 49, - -61, -65, - 39}, data); - - // chars above 0xff - ser = serpent.serialize("\u0100\u20ac\u8899"); - data = strip_header(ser); - // '\xc4\x80\xe2\x82\xac\xe8\xa2\x99' (has some utf-8 encoded chars in it) - assertArrayEquals(new byte[] {39, -60, -128, -30, -126, -84, -24, -94, -103, 39}, data); - - // some random high chars that are all printable in python and not escaped - ser = serpent.serialize("\u0377\u082d\u10c5\u135d\uac00"); - data = strip_header(ser); - // '\xcd\xb7\xe0\xa0\xad\xe1\x83\x85\xe1\x8d\x9d\xea\xb0\x80' (only a bunch of utf-8 encoded chars) - assertArrayEquals(new byte[] {39, -51, -73, -32, -96, -83, -31, -125, -123, -31, -115, -99, -22, -80, -128, 39}, data); - - // some random high chars that are all non-printable in python and that are escaped - ser = serpent.serialize("\u0378\u082e\u10c6\u135c\uabff"); - data = strip_header(ser); - // '\\u0378\\u082e\\u10c6\\u135c\\uabff' - assertArrayEquals(new byte[] {39, - 92, 117, 48, 51, 55, 56, - 92, 117, 48, 56, 50, 101, - 92, 117, 49, 48, 99, 54, - 92, 117, 49, 51, 53, 99, - 92, 117, 97, 98, 102, 102, - 39}, data); - } - - @Test - public void testNullByte() - { - Serializer serpent = new Serializer(); - byte[] ser = serpent.serialize("null\u0000byte"); - byte[] data = strip_header(ser); - assertEquals("'null\\x00byte'", new String(data)); - for(byte b: ser) { - if(b==0) - fail("serialized data may not contain 0-bytes"); - } - } - - @Test - public void testBool() - { - Serializer serpent = new Serializer(); - byte[] ser = serpent.serialize(true); - byte[] data = strip_header(ser); - assertEquals("True", S(data)); - ser = serpent.serialize(false); - data = strip_header(ser); - assertEquals("False", S(data)); - } - - - @Test - public void testBytes() - { - Serializer serpent = new Serializer(true, true, false); - byte[] bytes = new byte[] { 97, 98, 99, 100, 101, 102 }; // abcdef - byte[] ser = serpent.serialize(bytes); - assertEquals("{\n 'data': 'YWJjZGVm',\n 'encoding': 'base64'\n}", S(strip_header(ser))); - - Parser p = new Parser(); - String parsed = p.parse(ser).root.toString(); - assertEquals(39, parsed.length()); - - Map dict = new HashMap(); - dict.put("data", "YWJjZGVm"); - dict.put("encoding", "base64"); - - byte[] bytes2 = Parser.toBytes(dict); - assertArrayEquals(bytes, bytes2); - - dict.put("encoding", "base99"); - try { - Parser.toBytes(dict); - fail("error expected"); - } catch (IllegalArgumentException x) { - // - } - - dict.clear(); - try { - Parser.toBytes(dict); - fail("error expected"); - } catch (IllegalArgumentException x) { - // - } - dict.clear(); - dict.put("data", "YWJjZGVm"); - try { - Parser.toBytes(dict); - fail("error expected"); - } catch (IllegalArgumentException x) { - // - } - dict.clear(); - dict.put("encoding", "base64"); - try { - Parser.toBytes(dict); - fail("error expected"); - } catch (IllegalArgumentException x) { - // - } - try { - Parser.toBytes(12345); - fail("error expected"); - } catch (IllegalArgumentException x) { - // - } - try { - Parser.toBytes(null); - fail("error expected"); - } catch (IllegalArgumentException x) { - // - } - } - - - @Test - public void testDateTime() - { - Serializer serpent = new Serializer(); - Calendar cal = new GregorianCalendar(2013, 0, 20, 23, 59, 45); - cal.set(Calendar.MILLISECOND, 999); - cal.setTimeZone(TimeZone.getTimeZone("GMT+0")); - - byte[] ser = strip_header(serpent.serialize(cal)); - assertEquals("'2013-01-20T23:59:45.999Z'", S(ser)); - - Date date = cal.getTime(); - ser = strip_header(serpent.serialize(date)); - assertEquals("'2013-01-20T23:59:45.999Z'", S(ser)); - - cal.set(Calendar.MILLISECOND, 0); - ser = strip_header(serpent.serialize(cal)); - assertEquals("'2013-01-20T23:59:45Z'", S(ser)); - } - - @Test - public void testDateTimeWithTimezone() - { - Serializer serpent = new Serializer(); - Calendar cal = new GregorianCalendar(2013, 0, 20, 23, 59, 45); - cal.set(Calendar.MILLISECOND, 999); - cal.setTimeZone(TimeZone.getTimeZone("Europe/Amsterdam")); - - byte[] ser = strip_header(serpent.serialize(cal)); - assertEquals("'2013-01-20T23:59:45.999+01:00'", S(ser)); // normal time - - cal = new GregorianCalendar(2013, 4, 10, 13, 59, 45); - cal.set(Calendar.MILLISECOND, 999); - cal.setTimeZone(TimeZone.getTimeZone("Europe/Amsterdam")); - - ser = strip_header(serpent.serialize(cal)); - assertEquals("'2013-05-10T13:59:45.999+02:00'", S(ser)); // daylight saving time - - Date date=cal.getTime(); - ser = strip_header(serpent.serialize(date)); - assertEquals("'2013-05-10T11:59:45.999Z'", S(ser)); // the date and time in UTC - } - - @Test - public void testNumbers() - { - Serializer serpent = new Serializer(); - byte[] ser = serpent.serialize((int)12345); - byte[] data = strip_header(ser); - assertEquals("12345", S(data)); - ser = serpent.serialize((long)1234567891234567891L); - data = strip_header(ser); - assertEquals("1234567891234567891", S(data)); - ser = serpent.serialize(99.1234); - data = strip_header(ser); - assertEquals("99.1234", S(data)); - ser = serpent.serialize(new BigInteger("1234999999999912345678901234567890")); - data = strip_header(ser); - assertEquals("1234999999999912345678901234567890", S(data)); - ser = serpent.serialize(new BigDecimal("123456789.987654321987654321987654321987654321")); - data=strip_header(ser); - assertEquals("'123456789.987654321987654321987654321987654321'", S(data)); - ComplexNumber cplx = new ComplexNumber(2.2, 3.3); - ser = serpent.serialize(cplx); - data = strip_header(ser); - assertEquals("(2.2+3.3j)", S(data)); - cplx = new ComplexNumber(0, 3); - ser = serpent.serialize(cplx); - data = strip_header(ser); - assertEquals("(0.0+3.0j)", S(data)); - cplx = new ComplexNumber(-2, -3); - ser = serpent.serialize(cplx); - data = strip_header(ser); - assertEquals("(-2.0-3.0j)", S(data)); - cplx = new ComplexNumber(-2.5, -3.9); - ser = serpent.serialize(cplx); - data = strip_header(ser); - assertEquals("(-2.5-3.9j)", S(data)); - } - - @Test - public void testDoubleNanInf() - { - Serializer serpent = new Serializer(); - Object[] doubles = new Object[] {Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY, Double.NaN, - Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY, Float.NaN, - new ComplexNumber(Double.POSITIVE_INFINITY, 3.3)}; - byte[] ser = serpent.serialize(doubles); - byte[] data = strip_header(ser); - assertEquals("(1e30000,-1e30000,{'__class__':'float','value':'nan'},1e30000,-1e30000,{'__class__':'float','value':'nan'},(1e30000+3.3j))", S(data)); - } - - @Test - public void testList() - { - Serializer serpent = new Serializer(); - List list = new LinkedList(); - - // test empty list - byte[] ser = strip_header(serpent.serialize(list)); - assertEquals("[]", S(ser)); - serpent.indent=true; - ser = strip_header(serpent.serialize(list)); - assertEquals("[]", S(ser)); - serpent.indent=false; - - // test nonempty list - list.add(42); - list.add("Sally"); - list.add(16.5); - ser = strip_header(serpent.serialize(list)); - assertEquals("[42,'Sally',16.5]", S(ser)); - serpent.indent=true; - ser = strip_header(serpent.serialize(list)); - assertEquals("[\n 42,\n 'Sally',\n 16.5\n]", S(ser)); - } - - public class UnserializableClass - { - } - - @Test(expected = IllegalArgumentException.class) - public void testClassFail() - { - Serializer serpent = new Serializer(true, true, false); - Object obj = new UnserializableClass(); - serpent.serialize(obj); - } - - @Test - public void testClassOk() - { - Serializer.registerClass(SerializationHelperClass.class, null); - Serializer serpent = new Serializer(true, true, false); - SerializationHelperClass obj = new SerializationHelperClass(); - obj.i=99; - obj.s="hi"; - obj.x=42; - byte[] ser = strip_header(serpent.serialize(obj)); - assertEquals("{\n 'NUMBER': 42,\n '__class__': 'SerializationHelperClass',\n 'object': None,\n 'theInteger': 99,\n 'theString': 'hi',\n 'thingy': True,\n 'x': 'X'\n}", S(ser)); - } - - @Test - public void testClassPackageOk() - { - Serializer.registerClass(SerializationHelperClass.class, null); - Serializer serpent = new Serializer(true, true, true); - SerializationHelperClass obj = new SerializationHelperClass(); - obj.i=99; - obj.s="hi"; - obj.x=42; - byte[] ser = strip_header(serpent.serialize(obj)); - assertEquals("{\n 'NUMBER': 42,\n '__class__': 'net.razorvine.serpent.test.SerializationHelperClass',\n 'object': None,\n 'theInteger': 99,\n 'theString': 'hi',\n 'thingy': True,\n 'x': 'X'\n}", S(ser)); - } - - class TestclassConverter implements IClassSerializer - { - @Override - public Map convert(Object obj) { - SerializationHelperClass o = (SerializationHelperClass) obj; - Map result = new HashMap(); - result.put("__class@__", o.getClass().getSimpleName()+"@"); - result.put("i@", o.i); - result.put("s@", o.s); - result.put("x@", o.x); - return result; - } - } - - class ExceptionConverter implements IClassSerializer - { - @Override - public Map convert(Object obj) { - IllegalArgumentException e = (IllegalArgumentException) obj; - Map result = new HashMap(); - result.put("__class@__", e.getClass().getSimpleName()); - result.put("msg@", e.getMessage()); - return result; - } - } - - @Test - public void testCustomClassDict() - { - Serializer.registerClass(SerializationHelperClass.class, new TestclassConverter()); - Serializer serpent = new Serializer(true, true, false); - - SerializationHelperClass obj = new SerializationHelperClass(); - obj.i=99; - obj.s="hi"; - obj.x=42; - - byte[] ser = strip_header(serpent.serialize(obj)); - assertEquals("{\n '__class@__': 'SerializationHelperClass@',\n 'i@': 99,\n 's@': 'hi',\n 'x@': 42\n}", S(ser)); - } - - @Test - public void testCustomExceptionDict() - { - Serializer.registerClass(IllegalArgumentException.class, new ExceptionConverter()); - Serializer serpent = new Serializer(true, true, false); - - Exception x = new IllegalArgumentException("errormessage"); - byte[] ser = strip_header(serpent.serialize(x)); - assertEquals("{\n '__class@__': 'IllegalArgumentException',\n 'msg@': 'errormessage'\n}", S(ser)); - } - - @Test - public void testSet() - { - Serializer serpent = new Serializer(); - Set set = new HashSet(); - - // test empty set - byte[] ser = strip_header(serpent.serialize(set)); - assertEquals("()", S(ser)); // empty set is serialized as a tuple. - serpent.indent=true; - ser = strip_header(serpent.serialize(set)); - assertEquals("()", S(ser)); // empty set is serialized as a tuple. - serpent.indent=false; - - // test nonempty set - set.add("X"); - set.add("Sally"); - set.add("Y"); - ser = strip_header(serpent.serialize(set)); - assertEquals(17, ser.length); - assertTrue(S(ser).contains("'Sally'")); - assertTrue(S(ser).contains("'X'")); - assertTrue(S(ser).contains("'Y'")); - serpent.indent=true; - ser = strip_header(serpent.serialize(set)); - assertEquals("{\n 'Sally',\n 'X',\n 'Y'\n}", S(ser)); - - // test no set literals - serpent.indent=false; - serpent.setliterals=false; - ser = strip_header(serpent.serialize(set)); - assertEquals(17, ser.length); - assertTrue(S(ser).contains("'Sally'")); - assertTrue(S(ser).contains("'X'")); - assertTrue(S(ser).contains("'Y'")); - assertTrue(ser[0]=='('); - assertTrue(ser[ser.length-1]==')'); - } - - @Test - public void testCollection() - { - Collection intlist = new LinkedList(); - intlist.add(42); - intlist.add(43); - Serializer serpent = new Serializer(); - byte[] ser = serpent.serialize(intlist); - ser = strip_header(ser); - assertEquals("[42,43]", S(ser)); - - ser=strip_header(serpent.serialize(new int[] {42})); - assertEquals("(42,)", S(ser)); - ser=strip_header(serpent.serialize(new int[] {42, 43})); - assertEquals("(42,43)", S(ser)); - - serpent.indent=true; - ser = strip_header(serpent.serialize(intlist)); - assertEquals("[\n 42,\n 43\n]", S(ser)); - ser=strip_header(serpent.serialize(new int[] {42})); - assertEquals("(\n 42,\n)", S(ser)); - ser=strip_header(serpent.serialize(new int[] {42, 43})); - assertEquals("(\n 42,\n 43\n)", S(ser)); - } - - - @Test - public void testDictionary() - { - Serializer serpent = new Serializer(); - Parser p = new Parser(); - - // test empty dict - Hashtable ht = new Hashtable(); - byte[] ser = serpent.serialize(ht); - assertEquals("{}", S(strip_header(ser))); - - String parsed = p.parse(ser).root.toString(); - assertEquals("{}", parsed); - - //empty dict with indentation - serpent.indent=true; - ser = serpent.serialize(ht); - assertEquals("{}", S(strip_header(ser))); - - parsed = p.parse(ser).root.toString(); - assertEquals("{}", parsed); - - // test dict with values - serpent.indent=false; - ht = new Hashtable(); - ht.put(42, "fortytwo"); - ht.put("sixteen-and-half", 16.5); - ht.put("name", "Sally"); - ht.put("status", false); - - ser = serpent.serialize(ht); - assertEquals('}', ser[ser.length-1]); - assertTrue(ser[ser.length-2]!=','); - - parsed = p.parse(ser).root.toString(); - assertEquals(69, parsed.length()); - - // test indentation - serpent.indent=true; - ser = serpent.serialize(ht); - assertEquals('}', ser[ser.length-1]); - assertEquals('\n', ser[ser.length-2]); - assertTrue(ser[ser.length-3]!=','); - String ser_str = S(strip_header(ser)); - assertTrue(ser_str.contains("'name': 'Sally'")); - assertTrue(ser_str.contains("'status': False")); - assertTrue(ser_str.contains("42: 'fortytwo'")); - assertTrue(ser_str.contains("'sixteen-and-half': 16.5")); - - parsed = p.parse(ser).root.toString(); - assertEquals(69, parsed.length()); - - serpent.indent=false; - - // generic Dictionary test - Map mydict = new HashMap(); - mydict.put(1, "one"); - mydict.put(2, "two"); - ser = serpent.serialize(mydict); - ser_str = S(strip_header(ser)); - assertTrue(ser_str.equals("{2:'two',1:'one'}") || ser_str.equals("{1:'one',2:'two'}")); - } - - - @Test - public void testIndentation() - { - Map dict = new HashMap(); - List list = new LinkedList(); - list.add(1); - list.add(2); - list.add(new String[] {"a", "b"}); - dict.put("first", list); - - Map subdict = new HashMap(); - subdict.put(1, false); - dict.put("second", subdict); - - Set subset = new HashSet(); - subset.add(3); - subset.add(4); - dict.put("third", subset); - - Serializer serpent = new Serializer(); - serpent.indent=true; - byte[] ser = strip_header(serpent.serialize(dict)); - assertEquals("{\n"+ -" 'first': [\n"+ -" 1,\n"+ -" 2,\n"+ -" (\n"+ -" 'a',\n"+ -" 'b'\n"+ -" )\n"+ -" ],\n"+ -" 'second': {\n"+ -" 1: False\n"+ -" },\n"+ -" 'third': {\n"+ -" 3,\n"+ -" 4\n"+ -" }\n"+ -"}", S(ser)); - } - - @Test - public void testSorting() - { - Serializer serpent=new Serializer(); - ArrayList data1 = new ArrayList(); - data1.add(3); - data1.add(2); - data1.add(1); - byte[] ser = strip_header(serpent.serialize(data1)); - assertEquals("[3,2,1]", S(ser)); - int[] data2 = new int[] { 3,2,1 }; - ser = strip_header(serpent.serialize(data2)); - assertEquals("(3,2,1)", S(ser)); - - Set data3 = new HashSet(); - data3.add(42); - data3.add("hi"); - serpent.indent=true; - ser = strip_header(serpent.serialize(data3)); - assertTrue(S(ser).equals("{\n 42,\n 'hi'\n}") || S(ser).equals("{\n 'hi',\n 42\n}")); - - Map data4 = new HashMap(); - data4.put(5, "five"); - data4.put(3, "three"); - data4.put(1, "one"); - data4.put(4, "four"); - data4.put(2, "two"); - serpent.indent=true; - ser = strip_header(serpent.serialize(data4)); - assertEquals("{\n 1: 'one',\n 2: 'two',\n 3: 'three',\n 4: 'four',\n 5: 'five'\n}", S(ser)); - - Set data5 = new HashSet(); - data5.add("x"); - data5.add("y"); - data5.add("z"); - data5.add("c"); - data5.add("b"); - data5.add("a"); - serpent.indent=true; - ser = strip_header(serpent.serialize(data5)); - assertEquals("{\n 'a',\n 'b',\n 'c',\n 'x',\n 'y',\n 'z'\n}", S(ser)); - } - - interface IBaseInterface {}; - interface ISubInterface extends IBaseInterface {}; - class BaseClassWithInterface implements IBaseInterface, Serializable {}; - class SubClassWithInterface extends BaseClassWithInterface implements ISubInterface, Serializable {}; - class BaseClass implements Serializable {}; - class SubClass extends BaseClass implements Serializable {}; - abstract class AbstractBaseClass {}; - class ConcreteSubClass extends AbstractBaseClass implements Serializable {}; - - class AnyClassConverter implements IClassSerializer - { - @Override - public Map convert(Object obj) { - Map result = new HashMap(); - result.put("(SUB)CLASS", obj.getClass().getSimpleName()); - return result; - } - } - - @Test - public void testAbstractBaseClassHierarchyPickler() - { - ConcreteSubClass c = new ConcreteSubClass(); - Serializer serpent=new Serializer(); - byte[] data = serpent.serialize(c); - assertEquals("{'__class__':'ConcreteSubClass'}", S(strip_header(data))); // the default serializer - - Serializer.registerClass(AbstractBaseClass.class, new AnyClassConverter()); - data = serpent.serialize(c); - assertEquals("{'(SUB)CLASS':'ConcreteSubClass'}", S(strip_header(data))); // custom serializer - } - - @Test - public void testInterfaceHierarchyPickler() - { - BaseClassWithInterface b = new BaseClassWithInterface(); - SubClassWithInterface sub = new SubClassWithInterface(); - Serializer serpent=new Serializer(); - byte[] data = serpent.serialize(b); - assertEquals("{'__class__':'BaseClassWithInterface'}", S(strip_header(data))); // the default serializer - data = serpent.serialize(sub); - assertEquals("{'__class__':'SubClassWithInterface'}", S(strip_header(data))); // the default serializer - - Serializer.registerClass(IBaseInterface.class, new AnyClassConverter()); - data = serpent.serialize(b); - assertEquals("{'(SUB)CLASS':'BaseClassWithInterface'}", S(strip_header(data))); // custom serializer - data = serpent.serialize(sub); - assertEquals("{'(SUB)CLASS':'SubClassWithInterface'}", S(strip_header(data))); // custom serializer - } -} +/** + * Serpent, a Python literal expression serializer/deserializer + * (a.k.a. Python's ast.literal_eval in Java) + * Software license: "MIT software license". See http://opensource.org/licenses/MIT + * @author Irmen de Jong (irmen@razorvine.net) + */ + +package net.razorvine.serpent.test; + +import static org.junit.Assert.*; + +import java.io.Serializable; +import java.io.UnsupportedEncodingException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.*; + +import net.razorvine.serpent.ComplexNumber; +import net.razorvine.serpent.IClassSerializer; +import net.razorvine.serpent.Parser; +import net.razorvine.serpent.Serializer; + +import org.junit.Test; + +public class SerializeTest { + + public byte[] strip_header(byte[] data) + { + int start; + for(start=0; start=data.length) + { + throw new IllegalArgumentException("need header in string"); + } + start++; + byte[] result = new byte[data.length-start]; + System.arraycopy(data, start, result, 0, data.length-start); + return result; + } + + public byte[] B(String s) + { + try { + return s.getBytes("utf-8"); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + return null; + } + } + + public String S(byte[] b) + { + try { + return new String(b, "utf-8"); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + return null; + } + } + + + @Test + public void testHeader() + { + Serializer ser = new Serializer(); + byte[] data = ser.serialize(null); + assertEquals(35, data[0]); + String strdata = S(data); + assertEquals("# serpent utf-8 python3.2", strdata.split("\n")[0]); + + ser.setliterals=false; + data = ser.serialize(null); + strdata = S(data); + assertEquals("# serpent utf-8 python2.6", strdata.split("\n")[0]); + + data = B("# header\nfirst-line"); + data = strip_header(data); + assertEquals("first-line", S(data)); + } + + + @Test + public void testException() + { + Serializer.registerClass(IllegalArgumentException.class, null); + Exception x = new IllegalArgumentException("errormessage"); + Serializer serpent = new Serializer(true, true, false); + byte[] ser = strip_header(serpent.serialize(x)); + assertEquals("{\n '__class__': 'IllegalArgumentException',\n '__exception__': True,\n 'args': (\n 'errormessage',\n ),\n 'attributes': {}\n}", S(ser)); + } + + @Test + public void testExceptionPackage() + { + Serializer.registerClass(IllegalArgumentException.class, null); + Exception x = new IllegalArgumentException("errormessage"); + Serializer serpent = new Serializer(true, true, true); + byte[] ser = strip_header(serpent.serialize(x)); + assertEquals("{\n '__class__': 'java.lang.IllegalArgumentException',\n '__exception__': True,\n 'args': (\n 'errormessage',\n ),\n 'attributes': {}\n}", S(ser)); + } + + @Test + public void testStuff() + { + Serializer ser=new Serializer(); + byte[] result = ser.serialize("blerp"); + result=strip_header(result); + assertEquals("'blerp'", S(result)); + result = ser.serialize("\\"); + result=strip_header(result); + assertEquals("'\\\\'", S(result)); + result = ser.serialize(UUID.fromString("f1f8d00e-49a5-4662-ac1d-d5f0426ed293")); + result=strip_header(result); + assertEquals("'f1f8d00e-49a5-4662-ac1d-d5f0426ed293'", S(result)); + result = ser.serialize(new BigDecimal("123456789.987654321987654321987654321987654321")); + result=strip_header(result); + assertEquals("'123456789.987654321987654321987654321987654321'", S(result)); + } + + + @Test + public void testNull() + { + Serializer ser = new Serializer(); + byte[] data = ser.serialize(null); + data=strip_header(data); + assertEquals("None", S(data)); + } + + + @Test + public void testStrings() + { + Serializer serpent = new Serializer(); + byte[] ser = serpent.serialize("hello"); + byte[] data = strip_header(ser); + assertEquals("'hello'", S(data)); + ser = serpent.serialize("quotes'\""); + data = strip_header(ser); + assertEquals("'quotes\\'\"'", S(data)); + ser = serpent.serialize("quotes2'"); + data = strip_header(ser); + assertEquals("\"quotes2'\"", S(data)); + } + + @Test + public void testUnicodeEscapes() + { + Serializer serpent=new Serializer(); + + // regular escaped chars first + byte[] ser = serpent.serialize("\b\r\n\f\t \\"); + byte[] data = strip_header(ser); + // '\\x08\\r\\n\\x0c\\t \\\\' + assertArrayEquals(new byte[] {39, + 92, 120, 48, 56, + 92, 114, + 92, 110, + 92, 120, 48, 99, + 92, 116, + 32, + 92, 92, + 39}, data); + + // simple cases (chars < 0x80) + ser = serpent.serialize("\u0000\u0001\u001f\u007f"); + data = strip_header(ser); + // '\\x00\\x01\\x1f\\x7f' + assertArrayEquals(new byte[] {39, + 92, 120, 48, 48, + 92, 120, 48, 49, + 92, 120, 49, 102, + 92, 120, 55, 102, + 39 }, data); + + // chars 0x80 .. 0xff + ser = serpent.serialize("\u0080\u0081\u00ff"); + data = strip_header(ser); + // '\\x80\\x81\xc3\xbf' (has some utf-8 encoded chars in it) + assertArrayEquals(new byte[] {39, + 92, 120, 56, 48, + 92, 120, 56, 49, + -61, -65, + 39}, data); + + // chars above 0xff + ser = serpent.serialize("\u0100\u20ac\u8899"); + data = strip_header(ser); + // '\xc4\x80\xe2\x82\xac\xe8\xa2\x99' (has some utf-8 encoded chars in it) + assertArrayEquals(new byte[] {39, -60, -128, -30, -126, -84, -24, -94, -103, 39}, data); + + // some random high chars that are all printable in python and not escaped + ser = serpent.serialize("\u0377\u082d\u10c5\u135d\uac00"); + data = strip_header(ser); + // '\xcd\xb7\xe0\xa0\xad\xe1\x83\x85\xe1\x8d\x9d\xea\xb0\x80' (only a bunch of utf-8 encoded chars) + assertArrayEquals(new byte[] {39, -51, -73, -32, -96, -83, -31, -125, -123, -31, -115, -99, -22, -80, -128, 39}, data); + + // some random high chars that are all non-printable in python and that are escaped + ser = serpent.serialize("\u0378\u082e\u10c6\u135c\uabff"); + data = strip_header(ser); + // '\\u0378\\u082e\\u10c6\\u135c\\uabff' + assertArrayEquals(new byte[] {39, + 92, 117, 48, 51, 55, 56, + 92, 117, 48, 56, 50, 101, + 92, 117, 49, 48, 99, 54, + 92, 117, 49, 51, 53, 99, + 92, 117, 97, 98, 102, 102, + 39}, data); + } + + @Test + public void testNullByte() + { + Serializer serpent = new Serializer(); + byte[] ser = serpent.serialize("null\u0000byte"); + byte[] data = strip_header(ser); + assertEquals("'null\\x00byte'", new String(data)); + for(byte b: ser) { + if(b==0) + fail("serialized data may not contain 0-bytes"); + } + } + + @Test + public void testBool() + { + Serializer serpent = new Serializer(); + byte[] ser = serpent.serialize(true); + byte[] data = strip_header(ser); + assertEquals("True", S(data)); + ser = serpent.serialize(false); + data = strip_header(ser); + assertEquals("False", S(data)); + } + + + @Test + public void testBytes() + { + Serializer serpent = new Serializer(true, true, false); + byte[] bytes = new byte[] { 97, 98, 99, 100, 101, 102 }; // abcdef + byte[] ser = serpent.serialize(bytes); + assertEquals("{\n 'data': 'YWJjZGVm',\n 'encoding': 'base64'\n}", S(strip_header(ser))); + + Parser p = new Parser(); + String parsed = p.parse(ser).root.toString(); + assertEquals(39, parsed.length()); + + Map dict = new HashMap(); + dict.put("data", "YWJjZGVm"); + dict.put("encoding", "base64"); + + byte[] bytes2 = Parser.toBytes(dict); + assertArrayEquals(bytes, bytes2); + + dict.put("encoding", "base99"); + try { + Parser.toBytes(dict); + fail("error expected"); + } catch (IllegalArgumentException x) { + // + } + + dict.clear(); + try { + Parser.toBytes(dict); + fail("error expected"); + } catch (IllegalArgumentException x) { + // + } + dict.clear(); + dict.put("data", "YWJjZGVm"); + try { + Parser.toBytes(dict); + fail("error expected"); + } catch (IllegalArgumentException x) { + // + } + dict.clear(); + dict.put("encoding", "base64"); + try { + Parser.toBytes(dict); + fail("error expected"); + } catch (IllegalArgumentException x) { + // + } + try { + Parser.toBytes(12345); + fail("error expected"); + } catch (IllegalArgumentException x) { + // + } + try { + Parser.toBytes(null); + fail("error expected"); + } catch (IllegalArgumentException x) { + // + } + } + + + @Test + public void testDateTime() + { + Serializer serpent = new Serializer(); + Calendar cal = new GregorianCalendar(2013, 0, 20, 23, 59, 45); + cal.set(Calendar.MILLISECOND, 999); + cal.setTimeZone(TimeZone.getTimeZone("GMT+0")); + + byte[] ser = strip_header(serpent.serialize(cal)); + assertEquals("'2013-01-20T23:59:45.999Z'", S(ser)); + + Date date = cal.getTime(); + ser = strip_header(serpent.serialize(date)); + assertEquals("'2013-01-20T23:59:45.999Z'", S(ser)); + + ser = strip_header(serpent.serialize(date)); + assertEquals("'2013-01-20T23:59:45.999Z'", S(ser)); + + cal.set(Calendar.MILLISECOND, 0); + ser = strip_header(serpent.serialize(cal)); + assertEquals("'2013-01-20T23:59:45Z'", S(ser)); + + date = cal.getTime(); + ser = strip_header(serpent.serialize(date)); + assertEquals("'2013-01-20T23:59:45Z'", S(ser)); + } + + @Test + public void testDateTimeWithTimezone() + { + Serializer serpent = new Serializer(); + Calendar cal = new GregorianCalendar(2013, 0, 20, 23, 59, 45); + cal.set(Calendar.MILLISECOND, 999); + cal.setTimeZone(TimeZone.getTimeZone("Europe/Amsterdam")); + + byte[] ser = strip_header(serpent.serialize(cal)); + assertEquals("'2013-01-20T23:59:45.999+0100'", S(ser)); // normal time + + cal = new GregorianCalendar(2013, 4, 10, 13, 59, 45); + cal.set(Calendar.MILLISECOND, 999); + cal.setTimeZone(TimeZone.getTimeZone("Europe/Amsterdam")); + + ser = strip_header(serpent.serialize(cal)); + assertEquals("'2013-05-10T13:59:45.999+0200'", S(ser)); // daylight saving time + + Date date=cal.getTime(); + ser = strip_header(serpent.serialize(date)); + assertEquals("'2013-05-10T11:59:45.999Z'", S(ser)); // the date and time in UTC + + cal.set(Calendar.MILLISECOND, 0); + date=cal.getTime(); + ser = strip_header(serpent.serialize(date)); + assertEquals("'2013-05-10T11:59:45Z'", S(ser)); // the date and time in UTC + } + + @Test + public void testNumbers() + { + Serializer serpent = new Serializer(); + byte[] ser = serpent.serialize((int)12345); + byte[] data = strip_header(ser); + assertEquals("12345", S(data)); + ser = serpent.serialize((long)1234567891234567891L); + data = strip_header(ser); + assertEquals("1234567891234567891", S(data)); + ser = serpent.serialize(99.1234); + data = strip_header(ser); + assertEquals("99.1234", S(data)); + ser = serpent.serialize(new BigInteger("1234999999999912345678901234567890")); + data = strip_header(ser); + assertEquals("1234999999999912345678901234567890", S(data)); + ser = serpent.serialize(new BigDecimal("123456789.987654321987654321987654321987654321")); + data=strip_header(ser); + assertEquals("'123456789.987654321987654321987654321987654321'", S(data)); + ComplexNumber cplx = new ComplexNumber(2.2, 3.3); + ser = serpent.serialize(cplx); + data = strip_header(ser); + assertEquals("(2.2+3.3j)", S(data)); + cplx = new ComplexNumber(0, 3); + ser = serpent.serialize(cplx); + data = strip_header(ser); + assertEquals("(0.0+3.0j)", S(data)); + cplx = new ComplexNumber(-2, -3); + ser = serpent.serialize(cplx); + data = strip_header(ser); + assertEquals("(-2.0-3.0j)", S(data)); + cplx = new ComplexNumber(-2.5, -3.9); + ser = serpent.serialize(cplx); + data = strip_header(ser); + assertEquals("(-2.5-3.9j)", S(data)); + } + + @Test + public void testDoubleNanInf() + { + Serializer serpent = new Serializer(); + Object[] doubles = new Object[] {Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY, Double.NaN, + Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY, Float.NaN, + new ComplexNumber(Double.POSITIVE_INFINITY, 3.3)}; + byte[] ser = serpent.serialize(doubles); + byte[] data = strip_header(ser); + assertEquals("(1e30000,-1e30000,{'__class__':'float','value':'nan'},1e30000,-1e30000,{'__class__':'float','value':'nan'},(1e30000+3.3j))", S(data)); + } + + @Test + public void testList() + { + Serializer serpent = new Serializer(); + List list = new LinkedList(); + + // test empty list + byte[] ser = strip_header(serpent.serialize(list)); + assertEquals("[]", S(ser)); + serpent.indent=true; + ser = strip_header(serpent.serialize(list)); + assertEquals("[]", S(ser)); + serpent.indent=false; + + // test nonempty list + list.add(42); + list.add("Sally"); + list.add(16.5); + ser = strip_header(serpent.serialize(list)); + assertEquals("[42,'Sally',16.5]", S(ser)); + serpent.indent=true; + ser = strip_header(serpent.serialize(list)); + assertEquals("[\n 42,\n 'Sally',\n 16.5\n]", S(ser)); + } + + public class UnserializableClass + { + } + + @Test(expected = IllegalArgumentException.class) + public void testClassFail() + { + Serializer serpent = new Serializer(true, true, false); + Object obj = new UnserializableClass(); + serpent.serialize(obj); + } + + @Test + public void testClassOk() + { + Serializer.registerClass(SerializationHelperClass.class, null); + Serializer serpent = new Serializer(true, true, false); + SerializationHelperClass obj = new SerializationHelperClass(); + obj.i=99; + obj.s="hi"; + obj.x=42; + byte[] ser = strip_header(serpent.serialize(obj)); + assertEquals("{\n 'NUMBER': 42,\n '__class__': 'SerializationHelperClass',\n 'object': None,\n 'theInteger': 99,\n 'theString': 'hi',\n 'thingy': True,\n 'x': 'X'\n}", S(ser)); + } + + @Test + public void testClassPackageOk() + { + Serializer.registerClass(SerializationHelperClass.class, null); + Serializer serpent = new Serializer(true, true, true); + SerializationHelperClass obj = new SerializationHelperClass(); + obj.i=99; + obj.s="hi"; + obj.x=42; + byte[] ser = strip_header(serpent.serialize(obj)); + assertEquals("{\n 'NUMBER': 42,\n '__class__': 'net.razorvine.serpent.test.SerializationHelperClass',\n 'object': None,\n 'theInteger': 99,\n 'theString': 'hi',\n 'thingy': True,\n 'x': 'X'\n}", S(ser)); + } + + class TestclassConverter implements IClassSerializer + { + @Override + public Map convert(Object obj) { + SerializationHelperClass o = (SerializationHelperClass) obj; + Map result = new HashMap(); + result.put("__class@__", o.getClass().getSimpleName()+"@"); + result.put("i@", o.i); + result.put("s@", o.s); + result.put("x@", o.x); + return result; + } + } + + class ExceptionConverter implements IClassSerializer + { + @Override + public Map convert(Object obj) { + IllegalArgumentException e = (IllegalArgumentException) obj; + Map result = new HashMap(); + result.put("__class@__", e.getClass().getSimpleName()); + result.put("msg@", e.getMessage()); + return result; + } + } + + @Test + public void testCustomClassDict() + { + Serializer.registerClass(SerializationHelperClass.class, new TestclassConverter()); + Serializer serpent = new Serializer(true, true, false); + + SerializationHelperClass obj = new SerializationHelperClass(); + obj.i=99; + obj.s="hi"; + obj.x=42; + + byte[] ser = strip_header(serpent.serialize(obj)); + assertEquals("{\n '__class@__': 'SerializationHelperClass@',\n 'i@': 99,\n 's@': 'hi',\n 'x@': 42\n}", S(ser)); + } + + @Test + public void testCustomExceptionDict() + { + Serializer.registerClass(IllegalArgumentException.class, new ExceptionConverter()); + Serializer serpent = new Serializer(true, true, false); + + Exception x = new IllegalArgumentException("errormessage"); + byte[] ser = strip_header(serpent.serialize(x)); + assertEquals("{\n '__class@__': 'IllegalArgumentException',\n 'msg@': 'errormessage'\n}", S(ser)); + } + + @Test + public void testSet() + { + Serializer serpent = new Serializer(); + Set set = new HashSet(); + + // test empty set + byte[] ser = strip_header(serpent.serialize(set)); + assertEquals("()", S(ser)); // empty set is serialized as a tuple. + serpent.indent=true; + ser = strip_header(serpent.serialize(set)); + assertEquals("()", S(ser)); // empty set is serialized as a tuple. + serpent.indent=false; + + // test nonempty set + set.add("X"); + set.add("Sally"); + set.add("Y"); + ser = strip_header(serpent.serialize(set)); + assertEquals(17, ser.length); + assertTrue(S(ser).contains("'Sally'")); + assertTrue(S(ser).contains("'X'")); + assertTrue(S(ser).contains("'Y'")); + serpent.indent=true; + ser = strip_header(serpent.serialize(set)); + assertEquals("{\n 'Sally',\n 'X',\n 'Y'\n}", S(ser)); + + // test no set literals + serpent.indent=false; + serpent.setliterals=false; + ser = strip_header(serpent.serialize(set)); + assertEquals(17, ser.length); + assertTrue(S(ser).contains("'Sally'")); + assertTrue(S(ser).contains("'X'")); + assertTrue(S(ser).contains("'Y'")); + assertTrue(ser[0]=='('); + assertTrue(ser[ser.length-1]==')'); + } + + @Test + public void testCollection() + { + Collection intlist = new LinkedList(); + intlist.add(42); + intlist.add(43); + Serializer serpent = new Serializer(); + byte[] ser = serpent.serialize(intlist); + ser = strip_header(ser); + assertEquals("[42,43]", S(ser)); + + ser=strip_header(serpent.serialize(new int[] {42})); + assertEquals("(42,)", S(ser)); + ser=strip_header(serpent.serialize(new int[] {42, 43})); + assertEquals("(42,43)", S(ser)); + + serpent.indent=true; + ser = strip_header(serpent.serialize(intlist)); + assertEquals("[\n 42,\n 43\n]", S(ser)); + ser=strip_header(serpent.serialize(new int[] {42})); + assertEquals("(\n 42,\n)", S(ser)); + ser=strip_header(serpent.serialize(new int[] {42, 43})); + assertEquals("(\n 42,\n 43\n)", S(ser)); + } + + + @Test + public void testDictionary() + { + Serializer serpent = new Serializer(); + Parser p = new Parser(); + + // test empty dict + Hashtable ht = new Hashtable(); + byte[] ser = serpent.serialize(ht); + assertEquals("{}", S(strip_header(ser))); + + String parsed = p.parse(ser).root.toString(); + assertEquals("{}", parsed); + + //empty dict with indentation + serpent.indent=true; + ser = serpent.serialize(ht); + assertEquals("{}", S(strip_header(ser))); + + parsed = p.parse(ser).root.toString(); + assertEquals("{}", parsed); + + // test dict with values + serpent.indent=false; + ht = new Hashtable(); + ht.put(42, "fortytwo"); + ht.put("sixteen-and-half", 16.5); + ht.put("name", "Sally"); + ht.put("status", false); + + ser = serpent.serialize(ht); + assertEquals('}', ser[ser.length-1]); + assertTrue(ser[ser.length-2]!=','); + + parsed = p.parse(ser).root.toString(); + assertEquals(69, parsed.length()); + + // test indentation + serpent.indent=true; + ser = serpent.serialize(ht); + assertEquals('}', ser[ser.length-1]); + assertEquals('\n', ser[ser.length-2]); + assertTrue(ser[ser.length-3]!=','); + String ser_str = S(strip_header(ser)); + assertTrue(ser_str.contains("'name': 'Sally'")); + assertTrue(ser_str.contains("'status': False")); + assertTrue(ser_str.contains("42: 'fortytwo'")); + assertTrue(ser_str.contains("'sixteen-and-half': 16.5")); + + parsed = p.parse(ser).root.toString(); + assertEquals(69, parsed.length()); + + serpent.indent=false; + + // generic Dictionary test + Map mydict = new HashMap(); + mydict.put(1, "one"); + mydict.put(2, "two"); + ser = serpent.serialize(mydict); + ser_str = S(strip_header(ser)); + assertTrue(ser_str.equals("{2:'two',1:'one'}") || ser_str.equals("{1:'one',2:'two'}")); + } + + + @Test + public void testIndentation() + { + Map dict = new HashMap(); + List list = new LinkedList(); + list.add(1); + list.add(2); + list.add(new String[] {"a", "b"}); + dict.put("first", list); + + Map subdict = new HashMap(); + subdict.put(1, false); + dict.put("second", subdict); + + Set subset = new HashSet(); + subset.add(3); + subset.add(4); + dict.put("third", subset); + + Serializer serpent = new Serializer(); + serpent.indent=true; + byte[] ser = strip_header(serpent.serialize(dict)); + assertEquals("{\n"+ +" 'first': [\n"+ +" 1,\n"+ +" 2,\n"+ +" (\n"+ +" 'a',\n"+ +" 'b'\n"+ +" )\n"+ +" ],\n"+ +" 'second': {\n"+ +" 1: False\n"+ +" },\n"+ +" 'third': {\n"+ +" 3,\n"+ +" 4\n"+ +" }\n"+ +"}", S(ser)); + } + + @Test + public void testSorting() + { + Serializer serpent=new Serializer(); + ArrayList data1 = new ArrayList(); + data1.add(3); + data1.add(2); + data1.add(1); + byte[] ser = strip_header(serpent.serialize(data1)); + assertEquals("[3,2,1]", S(ser)); + int[] data2 = new int[] { 3,2,1 }; + ser = strip_header(serpent.serialize(data2)); + assertEquals("(3,2,1)", S(ser)); + + Set data3 = new HashSet(); + data3.add(42); + data3.add("hi"); + serpent.indent=true; + ser = strip_header(serpent.serialize(data3)); + assertTrue(S(ser).equals("{\n 42,\n 'hi'\n}") || S(ser).equals("{\n 'hi',\n 42\n}")); + + Map data4 = new HashMap(); + data4.put(5, "five"); + data4.put(3, "three"); + data4.put(1, "one"); + data4.put(4, "four"); + data4.put(2, "two"); + serpent.indent=true; + ser = strip_header(serpent.serialize(data4)); + assertEquals("{\n 1: 'one',\n 2: 'two',\n 3: 'three',\n 4: 'four',\n 5: 'five'\n}", S(ser)); + + Set data5 = new HashSet(); + data5.add("x"); + data5.add("y"); + data5.add("z"); + data5.add("c"); + data5.add("b"); + data5.add("a"); + serpent.indent=true; + ser = strip_header(serpent.serialize(data5)); + assertEquals("{\n 'a',\n 'b',\n 'c',\n 'x',\n 'y',\n 'z'\n}", S(ser)); + } + + interface IBaseInterface {}; + interface ISubInterface extends IBaseInterface {}; + class BaseClassWithInterface implements IBaseInterface, Serializable {}; + class SubClassWithInterface extends BaseClassWithInterface implements ISubInterface, Serializable {}; + class BaseClass implements Serializable {}; + class SubClass extends BaseClass implements Serializable {}; + abstract class AbstractBaseClass {}; + class ConcreteSubClass extends AbstractBaseClass implements Serializable {}; + + class AnyClassConverter implements IClassSerializer + { + @Override + public Map convert(Object obj) { + Map result = new HashMap(); + result.put("(SUB)CLASS", obj.getClass().getSimpleName()); + return result; + } + } + + @Test + public void testAbstractBaseClassHierarchyPickler() + { + ConcreteSubClass c = new ConcreteSubClass(); + Serializer serpent=new Serializer(); + byte[] data = serpent.serialize(c); + assertEquals("{'__class__':'ConcreteSubClass'}", S(strip_header(data))); // the default serializer + + Serializer.registerClass(AbstractBaseClass.class, new AnyClassConverter()); + data = serpent.serialize(c); + assertEquals("{'(SUB)CLASS':'ConcreteSubClass'}", S(strip_header(data))); // custom serializer + } + + @Test + public void testInterfaceHierarchyPickler() + { + BaseClassWithInterface b = new BaseClassWithInterface(); + SubClassWithInterface sub = new SubClassWithInterface(); + Serializer serpent=new Serializer(); + byte[] data = serpent.serialize(b); + assertEquals("{'__class__':'BaseClassWithInterface'}", S(strip_header(data))); // the default serializer + data = serpent.serialize(sub); + assertEquals("{'__class__':'SubClassWithInterface'}", S(strip_header(data))); // the default serializer + + Serializer.registerClass(IBaseInterface.class, new AnyClassConverter()); + data = serpent.serialize(b); + assertEquals("{'(SUB)CLASS':'BaseClassWithInterface'}", S(strip_header(data))); // custom serializer + data = serpent.serialize(sub); + assertEquals("{'(SUB)CLASS':'SubClassWithInterface'}", S(strip_header(data))); // custom serializer + } +} From 1439b0b5d817bbf59972980f119dde1004a366ab Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Mon, 3 Jul 2017 22:36:05 +0200 Subject: [PATCH 107/230] java version in readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 01ce5c7..a3d98bd 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,8 @@ JAVA Maven-artefact is available on maven central, groupid 'net.razorvine' artifactid 'serpent'. Full source code can be found in ./java/ directory. Example usage can be found in ./java/test/SerpentExample.java +Versions before 1.23 require Java 7 or Java 8 (JDK 1.7 or 1.8) to compile and run. +Version 1.23 and later require Java 8 (JDK 1.8) at a minimum to compile and run. SOME MORE DETAILS From aa5d604871c7dd8f2c98dedcbb3836c88b13b29b Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Mon, 3 Jul 2017 22:45:24 +0200 Subject: [PATCH 108/230] [maven-release-plugin] prepare release serpent-1.23 --- java/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index bd72eb2..3bd1ebc 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -9,7 +9,7 @@ net.razorvine serpent - 1.23-SNAPSHOT + 1.23 jar serpent @@ -68,7 +68,7 @@ It is an alternative to JSON to provide easy data integration between Java and P https://github.com/irmen/Serpent scm:git:https://github.com/irmen/Serpent.git scm:git:https://github.com/irmen/Serpent.git - HEAD + serpent-1.23 Github From ab86d262f64872c033578d88a714efde47dc6e6c Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Mon, 3 Jul 2017 22:45:29 +0200 Subject: [PATCH 109/230] [maven-release-plugin] prepare for next development iteration --- java/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 3bd1ebc..0557e69 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -9,7 +9,7 @@ net.razorvine serpent - 1.23 + 1.24-SNAPSHOT jar serpent @@ -68,7 +68,7 @@ It is an alternative to JSON to provide easy data integration between Java and P https://github.com/irmen/Serpent scm:git:https://github.com/irmen/Serpent.git scm:git:https://github.com/irmen/Serpent.git - serpent-1.23 + HEAD Github From 8fdf0e01f4d5108fbe0ad6264f315d639f1c2890 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Fri, 11 Aug 2017 03:34:53 +0200 Subject: [PATCH 110/230] newer travis image --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 2da28e0..2e30b4a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ python: # Use fast travis build infrastructure explicitly sudo: false +dist: trusty # Installation installs dependencies install: From 0971fcc8de663f572a33cd0de779dddb408e38ff Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sun, 5 Nov 2017 00:28:03 +0100 Subject: [PATCH 111/230] removed python 3.3 support (it is end of life) added python 3.7 support and tox config (release is approaching) --- setup.py | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index be962fd..bee28db 100755 --- a/setup.py +++ b/setup.py @@ -163,10 +163,10 @@ def __init__(self): "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", "Topic :: Software Development" ], diff --git a/tox.ini b/tox.ini index 76e5c85..03e6ccf 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=py27,py33,py34,py35,py36,pypy,pypy3 +envlist=py27,py34,py35,py36,py37,pypy,pypy3 [testenv] deps=pytz From 364c9f976e17c28d749344a1fb63f8b649a42454 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sun, 5 Nov 2017 00:30:11 +0100 Subject: [PATCH 112/230] travis python 3.7 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2e30b4a..ed0ca2b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,10 @@ language: python python: - "2.7" - - "3.3" - "3.4" - "3.5" - "3.6" + - "3.7" - "pypy" - "pypy3" From 9a00212c1fe68f6a558df594c88b93a8a9f8e895 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sun, 5 Nov 2017 00:32:38 +0100 Subject: [PATCH 113/230] travis 3.7-dev --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ed0ca2b..6fc1db9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ python: - "3.4" - "3.5" - "3.6" - - "3.7" + - "3.7-dev" - "pypy" - "pypy3" From e0d81baa053b7e635c776c8591040aaa8e71c153 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 14 Nov 2017 23:12:28 +0100 Subject: [PATCH 114/230] stricter python flags when running tests --- .travis.yml | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6fc1db9..82d3363 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,4 +17,4 @@ install: - pip install pytz - pip install . -script: cd tests && python -bb test_serpent.py +script: cd tests && python -E -Wall -tt -bb test_serpent.py diff --git a/tox.ini b/tox.ini index 03e6ccf..8332538 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ envlist=py27,py34,py35,py36,py37,pypy,pypy3 [testenv] deps=pytz changedir={toxinidir}/tests -commands=python -bb -E test_serpent.py +commands=python -E -Wall -tt -bb test_serpent.py [testenv:py26] deps=unittest2 From 3cdbdf7ab2b2feb7eed5b007453f1854e3a8391b Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 14 Nov 2017 23:41:01 +0100 Subject: [PATCH 115/230] stricter python flags when running tests --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 82d3363..1c89384 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,4 +17,7 @@ install: - pip install pytz - pip install . -script: cd tests && python -E -Wall -tt -bb test_serpent.py +script: + - if [[ $TRAVIS_PYTHON_VERSION != pypy* ]]; then cd tests && python -E -Wall -tt -bb test_serpent.py; fi + - if [[ $TRAVIS_PYTHON_VERSION == pypy* ]]; then cd tests && python -E -Wall -bb test_serpent.py; fi +# pypy doesn't like -tt option From bab2eac03f89924163d8ec7f83105923e2c7c9f9 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Tue, 14 Nov 2017 23:49:16 +0100 Subject: [PATCH 116/230] stricter python flags when running tests, fix invalid escape sequence in test --- tests/test_serpent.py | 4 ++-- tox.ini | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_serpent.py b/tests/test_serpent.py index 8355537..a5aacf8 100644 --- a/tests/test_serpent.py +++ b/tests/test_serpent.py @@ -77,9 +77,9 @@ def test_trailing_comma_set(self): self.assertEqual(set([1, 2, 3]), v) def test_unicode_escapes(self): - v = serpent.loads(b"'\u20ac'") + v = serpent.loads(b"'\\u20ac'") self.assertEqual(u"\u20ac", v) - v = serpent.loads(b"'\U00022001'") + v = serpent.loads(b"'\\U00022001'") self.assertEqual(u"\U00022001", v) def test_input_types(self): diff --git a/tox.ini b/tox.ini index 8332538..4a3d903 100644 --- a/tox.ini +++ b/tox.ini @@ -9,3 +9,7 @@ commands=python -E -Wall -tt -bb test_serpent.py [testenv:py26] deps=unittest2 pytz + +[testenv:pypy3] +commands=python -E -Wall -bb test_serpent.py +# pypy3 doesn't like the -tt option From fb117d4dac3464ad9c9b50461654e5cea60c6c84 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Thu, 16 Nov 2017 22:22:06 +0100 Subject: [PATCH 117/230] fixed file endings/encoding --- dotnet/Serpent.Test/CycleTest.cs | 12 +- dotnet/Serpent.Test/Example.cs | 186 +-- dotnet/Serpent.Test/ParserTest.cs | 1650 +++++++++++++------------- dotnet/Serpent.Test/SerializeTest.cs | 1486 +++++++++++------------ 4 files changed, 1667 insertions(+), 1667 deletions(-) diff --git a/dotnet/Serpent.Test/CycleTest.cs b/dotnet/Serpent.Test/CycleTest.cs index c02a1e5..f16ab58 100644 --- a/dotnet/Serpent.Test/CycleTest.cs +++ b/dotnet/Serpent.Test/CycleTest.cs @@ -24,7 +24,7 @@ public void testTupleOk() var d = new object[] {t,t,t}; var data = ser.Serialize(d); var parser = new Parser(); - var ast = parser.Parse(data); + parser.Parse(data); } [Test] @@ -41,7 +41,7 @@ public void testListOk() d.Add(t); var data = ser.Serialize(d); var parser = new Parser(); - var ast = parser.Parse(data); + parser.Parse(data); } [Test] @@ -56,7 +56,7 @@ public void testDictOk() d["z"] = t; var data = ser.Serialize(d); var parser = new Parser(); - var ast = parser.Parse(data); + parser.Parse(data); } [Test] @@ -68,7 +68,7 @@ public void testListCycle() d.Add(1); d.Add(2); d.Add(d); - var data = ser.Serialize(d); + ser.Serialize(d); } [Test] @@ -80,7 +80,7 @@ public void testDictCycle() d["x"] = 1; d["y"] = 2; d["z"] = d; - var data = ser.Serialize(d); + ser.Serialize(d); } [Test] @@ -93,7 +93,7 @@ public void testClassCycle() d.i = 99; d.s = "hello"; d.obj = d; - var data = ser.Serialize(d); + ser.Serialize(d); } [Test] diff --git a/dotnet/Serpent.Test/Example.cs b/dotnet/Serpent.Test/Example.cs index 2a11252..6d18372 100644 --- a/dotnet/Serpent.Test/Example.cs +++ b/dotnet/Serpent.Test/Example.cs @@ -1,93 +1,93 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.Text; -using System.Linq; -using System.IO; -using NUnit.Framework; -using Razorvine.Serpent; - -namespace Razorvine.Serpent.Test -{ - /// - /// Example usage. - /// - [TestFixture] - [Explicit("example")] - public class Example - { - [Test] - public void ExampleUsage() - { - Console.WriteLine("using serpent library version {0}", LibraryVersion.Version); - - var data = new Dictionary { - {"tuple", new int[] { 1,2,3 } }, - {"date", DateTime.Now}, - {"set", new HashSet { "a", "b", "c" } }, - {"class", new SampleClass() { - name = "Sally", - age = 26 - }} - }; - - // serialize data structure to bytes - Serializer serpent = new Serializer(indent: true); - byte[] ser = serpent.Serialize(data); - // print it on the screen, but normally you'd store byte bytes in a file or transfer them across a network connection - Console.WriteLine("Serialized:"); - Console.WriteLine(Encoding.UTF8.GetString(ser)); - - // parse the serialized bytes back into an abstract syntax tree of the datastructure - Parser parser = new Parser(); - Ast ast = parser.Parse(ser); - Console.WriteLine("\nParsed AST:"); - Console.WriteLine(ast.Root.ToString()); - - // print debug representation - DebugVisitor dv = new DebugVisitor(); - ast.Accept(dv); - Console.WriteLine("DEBUG string representation:"); - Console.WriteLine(dv.ToString()); - - // turn the Ast into regular .net objects - var dict = (IDictionary) ast.GetData(); - // You can get the data out of the Ast manually as well, by using the supplied visitor: - // var visitor = new ObjectifyVisitor(); - // ast.Accept(visitor); - // var dict = (IDictionary) visitor.GetObject(); - - // print the results - Console.WriteLine("PARSED results:"); - Console.Write("tuple items: "); - object[] tuple = (object[]) dict["tuple"]; - Console.WriteLine(string.Join(", ", tuple.Select(e=>e.ToString()).ToArray())); - Console.WriteLine("date: {0}", dict["date"]); - Console.Write("set items: "); - HashSet set = (HashSet) dict["set"]; - Console.WriteLine(string.Join(", ", set.Select(e=>e.ToString()).ToArray())); - Console.WriteLine("class attributes:"); - var clazz = (IDictionary) dict["class"]; // custom classes are serialized as dicts - Console.WriteLine(" type: {0}", clazz["__class__"]); - Console.WriteLine(" name: {0}", clazz["name"]); - Console.WriteLine(" age: {0}", clazz["age"]); - - Console.WriteLine(""); - - // parse and print the example file - ser=File.ReadAllBytes("testserpent.utf8.bin"); - ast = parser.Parse(ser); - dv = new DebugVisitor(); - ast.Accept(dv); - Console.WriteLine("DEBUG string representation of the test file:"); - Console.WriteLine(dv.ToString()); - } - - [Serializable] - public class SampleClass - { - public int age {get;set;} - public string name {get;set;} - } - } -} +using System; +using System.Collections; +using System.Collections.Generic; +using System.Text; +using System.Linq; +using System.IO; +using NUnit.Framework; +using Razorvine.Serpent; + +namespace Razorvine.Serpent.Test +{ + /// + /// Example usage. + /// + [TestFixture] + [Explicit("example")] + public class Example + { + [Test] + public void ExampleUsage() + { + Console.WriteLine("using serpent library version {0}", LibraryVersion.Version); + + var data = new Dictionary { + {"tuple", new int[] { 1,2,3 } }, + {"date", DateTime.Now}, + {"set", new HashSet { "a", "b", "c" } }, + {"class", new SampleClass() { + name = "Sally", + age = 26 + }} + }; + + // serialize data structure to bytes + Serializer serpent = new Serializer(indent: true); + byte[] ser = serpent.Serialize(data); + // print it on the screen, but normally you'd store byte bytes in a file or transfer them across a network connection + Console.WriteLine("Serialized:"); + Console.WriteLine(Encoding.UTF8.GetString(ser)); + + // parse the serialized bytes back into an abstract syntax tree of the datastructure + Parser parser = new Parser(); + Ast ast = parser.Parse(ser); + Console.WriteLine("\nParsed AST:"); + Console.WriteLine(ast.Root.ToString()); + + // print debug representation + DebugVisitor dv = new DebugVisitor(); + ast.Accept(dv); + Console.WriteLine("DEBUG string representation:"); + Console.WriteLine(dv.ToString()); + + // turn the Ast into regular .net objects + var dict = (IDictionary) ast.GetData(); + // You can get the data out of the Ast manually as well, by using the supplied visitor: + // var visitor = new ObjectifyVisitor(); + // ast.Accept(visitor); + // var dict = (IDictionary) visitor.GetObject(); + + // print the results + Console.WriteLine("PARSED results:"); + Console.Write("tuple items: "); + object[] tuple = (object[]) dict["tuple"]; + Console.WriteLine(string.Join(", ", tuple.Select(e=>e.ToString()).ToArray())); + Console.WriteLine("date: {0}", dict["date"]); + Console.Write("set items: "); + HashSet set = (HashSet) dict["set"]; + Console.WriteLine(string.Join(", ", set.Select(e=>e.ToString()).ToArray())); + Console.WriteLine("class attributes:"); + var clazz = (IDictionary) dict["class"]; // custom classes are serialized as dicts + Console.WriteLine(" type: {0}", clazz["__class__"]); + Console.WriteLine(" name: {0}", clazz["name"]); + Console.WriteLine(" age: {0}", clazz["age"]); + + Console.WriteLine(""); + + // parse and print the example file + ser=File.ReadAllBytes("testserpent.utf8.bin"); + ast = parser.Parse(ser); + dv = new DebugVisitor(); + ast.Accept(dv); + Console.WriteLine("DEBUG string representation of the test file:"); + Console.WriteLine(dv.ToString()); + } + + [Serializable] + public class SampleClass + { + public int age {get;set;} + public string name {get;set;} + } + } +} diff --git a/dotnet/Serpent.Test/ParserTest.cs b/dotnet/Serpent.Test/ParserTest.cs index 9f27521..bc59b1b 100644 --- a/dotnet/Serpent.Test/ParserTest.cs +++ b/dotnet/Serpent.Test/ParserTest.cs @@ -1,826 +1,826 @@ -/// -/// Serpent, a Python literal expression serializer/deserializer -/// (a.k.a. Python's ast.literal_eval in .NET) -/// -/// Copyright Irmen de Jong (irmen@razorvine.net) -/// Software license: "MIT software license". See http://opensource.org/licenses/MIT -/// - -using System; -using System.Collections; -using System.Collections.Generic; -using System.IO; -using System.Text; -using NUnit.Framework; - -namespace Razorvine.Serpent.Test -{ - [TestFixture] - public class ParserTest - { - [Test] - public void TestBasic() - { - Parser p = new Parser(); - Assert.IsNull(p.Parse((string)null).Root); - Assert.IsNull(p.Parse("").Root); - Assert.IsNotNull(p.Parse("# comment\n42\n").Root); - } - - [Test] - public void TestComments() - { - Parser p = new Parser(); - - Ast ast = p.Parse("[ 1, 2 ]"); // no header whatsoever - var visitor = new ObjectifyVisitor(); - ast.Accept(visitor); - Object obj = visitor.GetObject(); - Assert.AreEqual(new int[] {1,2}, obj); - - ast = p.Parse(@"# serpent utf-8 python2.7 -[ 1, 2, - # some comments here - 3, 4] # more here -# and here. -"); - visitor = new ObjectifyVisitor(); - ast.Accept(visitor); - obj = visitor.GetObject(); - Assert.AreEqual(new int[] {1,2,3,4}, obj); - } - - [Test] - public void TestPrimitives() - { - Parser p = new Parser(); - Assert.AreEqual(new Ast.IntegerNode(42), p.Parse("42").Root); - Assert.AreEqual(new Ast.IntegerNode(-42), p.Parse("-42").Root); - Assert.AreEqual(new Ast.DoubleNode(42.331), p.Parse("42.331").Root); - Assert.AreEqual(new Ast.DoubleNode(-42.331), p.Parse("-42.331").Root); - Assert.AreEqual(new Ast.DoubleNode(-1.2e19), p.Parse("-1.2e+19").Root); - Assert.AreEqual(new Ast.DoubleNode(-1.2e19), p.Parse("-1.2e19").Root); - Assert.AreEqual(new Ast.DoubleNode(0.0004), p.Parse("4e-4").Root); - Assert.AreEqual(new Ast.DoubleNode(40000), p.Parse("4e4").Root); - Assert.AreEqual(new Ast.BooleanNode(true), p.Parse("True").Root); - Assert.AreEqual(new Ast.BooleanNode(false), p.Parse("False").Root); - Assert.AreEqual(Ast.NoneNode.Instance, p.Parse("None").Root); - - // long ints - Assert.AreEqual(new Ast.DecimalNode(123456789123456789123456789M), p.Parse("123456789123456789123456789").Root); - Assert.AreNotEqual(new Ast.LongNode(52), p.Parse("52").Root); - Assert.AreEqual(new Ast.LongNode(123456789123456789L), p.Parse("123456789123456789").Root); - Assert.Throws(()=>p.Parse("123456789123456789123456789123456789")); // overflow - } - - [Test] - public void TestWeirdFloats() - { - Parser p = new Parser(); - var d = (Ast.DoubleNode) p.Parse("1e30000").Root; - Assert.IsTrue(double.IsPositiveInfinity(d.Value)); - d = (Ast.DoubleNode) p.Parse("-1e30000").Root; - Assert.IsTrue(double.IsNegativeInfinity(d.Value)); - - var tuple = (Ast.TupleNode) p.Parse("(1e30000,-1e30000,{'__class__':'float','value':'nan'})").Root; - Assert.AreEqual(3, tuple.Elements.Count); - d = (Ast.DoubleNode) tuple.Elements[0]; - Assert.IsTrue(double.IsPositiveInfinity(d.Value)); - d = (Ast.DoubleNode) tuple.Elements[1]; - Assert.IsTrue(double.IsNegativeInfinity(d.Value)); - d = (Ast.DoubleNode) tuple.Elements[2]; - Assert.IsTrue(Double.IsNaN(d.Value)); - - var c = (Ast.ComplexNumberNode) p.Parse("(1e30000-1e30000j)").Root; - Assert.IsTrue(double.IsPositiveInfinity(c.Real)); - Assert.IsTrue(double.IsNegativeInfinity(c.Imaginary)); - } - - [Test] - public void TestFloatPrecision() - { - Parser p = new Parser(); - Serializer serpent = new Serializer(); - var ser = serpent.Serialize(1.2345678987654321); - Ast.DoubleNode dv = (Ast.DoubleNode) p.Parse(ser).Root; - Assert.AreEqual(1.2345678987654321.ToString(), dv.Value.ToString()); - - ser = serpent.Serialize(5555.12345678987656); - dv = (Ast.DoubleNode) p.Parse(ser).Root; - Assert.AreEqual(5555.12345678987656.ToString(), dv.Value.ToString()); - - ser = serpent.Serialize(98765432123456.12345678987656); - dv = (Ast.DoubleNode) p.Parse(ser).Root; - Assert.AreEqual(98765432123456.12345678987656.ToString(), dv.Value.ToString()); - - ser = serpent.Serialize(98765432123456.12345678987656e+44); - dv = (Ast.DoubleNode) p.Parse(ser).Root; - Assert.AreEqual((98765432123456.12345678987656e+44).ToString(), dv.Value.ToString()); - - ser = serpent.Serialize(-98765432123456.12345678987656e-44); - dv = (Ast.DoubleNode) p.Parse(ser).Root; - Assert.AreEqual((-98765432123456.12345678987656e-44).ToString(), dv.Value.ToString()); - } - - [Test] - public void TestEquality() - { - Ast.INode n1, n2; - - n1 = new Ast.IntegerNode(42); - n2 = new Ast.IntegerNode(42); - Assert.AreEqual(n1, n2); - n2 = new Ast.IntegerNode(43); - Assert.AreNotEqual(n1, n2); - - n1 = new Ast.StringNode("foo"); - n2 = new Ast.StringNode("foo"); - Assert.AreEqual(n1, n2); - n2 = new Ast.StringNode("bar"); - Assert.AreNotEqual(n1, n2); - - n1 = new Ast.ComplexNumberNode() { - Real=1.1, - Imaginary=2.2 - }; - n2 = new Ast.ComplexNumberNode() { - Real=1.1, - Imaginary=2.2 - }; - Assert.AreEqual(n1, n2); - n2 = new Ast.ComplexNumberNode() { - Real=1.1, - Imaginary=3.3 - }; - Assert.AreNotEqual(n1, n2); - - n1=new Ast.KeyValueNode() { - Key=new Ast.IntegerNode(42), - Value=new Ast.IntegerNode(42) - }; - n2=new Ast.KeyValueNode() { - Key=new Ast.IntegerNode(42), - Value=new Ast.IntegerNode(42) - }; - Assert.AreEqual(n1, n2); - n1=new Ast.KeyValueNode() { - Key=new Ast.IntegerNode(43), - Value=new Ast.IntegerNode(43) - }; - Assert.AreNotEqual(n1,n2); - - n1=Ast.NoneNode.Instance; - n2=Ast.NoneNode.Instance; - Assert.AreEqual(n1, n2); - n2=new Ast.IntegerNode(42); - Assert.AreNotEqual(n1, n2); - - n1=new Ast.DictNode() { - Elements=new List() { - new Ast.KeyValueNode() { - Key=new Ast.IntegerNode(42), - Value=new Ast.IntegerNode(42) - } - } - }; - n2=new Ast.DictNode() { - Elements=new List() { - new Ast.KeyValueNode() { - Key=new Ast.IntegerNode(42), - Value=new Ast.IntegerNode(42) - } - } - }; - Assert.AreEqual(n1, n2); - n2=new Ast.DictNode() { - Elements=new List() { - new Ast.KeyValueNode() { - Key=new Ast.IntegerNode(42), - Value=new Ast.IntegerNode(43) - } - } - }; - Assert.AreNotEqual(n1, n2); - - n1=new Ast.ListNode() { - Elements=new List() { - new Ast.IntegerNode(42) - } - }; - n2=new Ast.ListNode() { - Elements=new List() { - new Ast.IntegerNode(42) - } - }; - Assert.AreEqual(n1,n2); - n2=new Ast.ListNode() { - Elements=new List() { - new Ast.IntegerNode(43) - } - }; - Assert.AreNotEqual(n1,n2); - - n1=new Ast.SetNode() { - Elements=new List() { - new Ast.IntegerNode(42) - } - }; - n2=new Ast.SetNode() { - Elements=new List() { - new Ast.IntegerNode(42) - } - }; - Assert.AreEqual(n1,n2); - n2=new Ast.SetNode() { - Elements=new List() { - new Ast.IntegerNode(43) - } - }; - Assert.AreNotEqual(n1,n2); - - n1=new Ast.TupleNode() { - Elements=new List() { - new Ast.IntegerNode(42) - } - }; - n2=new Ast.TupleNode() { - Elements=new List() { - new Ast.IntegerNode(42) - } - }; - Assert.AreEqual(n1,n2); - n2=new Ast.TupleNode() { - Elements=new List() { - new Ast.IntegerNode(43) - } - }; - Assert.AreNotEqual(n1,n2); - - } - - [Test] - public void TestDictEquality() - { - Ast.DictNode dict1 = new Ast.DictNode(); - Ast.KeyValueNode kv=new Ast.KeyValueNode() { - Key=new Ast.StringNode("key1"), - Value=new Ast.IntegerNode(42) - }; - dict1.Elements.Add(kv); - kv=new Ast.KeyValueNode() { - Key=new Ast.StringNode("key2"), - Value=new Ast.IntegerNode(43) - }; - dict1.Elements.Add(kv); - kv=new Ast.KeyValueNode() { - Key=new Ast.StringNode("key3"), - Value=new Ast.IntegerNode(44) - }; - dict1.Elements.Add(kv); - - Ast.DictNode dict2 = new Ast.DictNode(); - kv=new Ast.KeyValueNode(){ - Key=new Ast.StringNode("key2"), - Value=new Ast.IntegerNode(43) - }; - dict2.Elements.Add(kv); - kv=new Ast.KeyValueNode() { - Key=new Ast.StringNode("key3"), - Value=new Ast.IntegerNode(44) - }; - dict2.Elements.Add(kv); - kv=new Ast.KeyValueNode() { - Key=new Ast.StringNode("key1"), - Value=new Ast.IntegerNode(42) - }; - dict2.Elements.Add(kv); - - Assert.AreEqual(dict1, dict2); - kv=new Ast.KeyValueNode() { - Key=new Ast.StringNode("key4"), - Value=new Ast.IntegerNode(45) - }; - dict2.Elements.Add(kv); - Assert.AreNotEqual(dict1, dict2); - } - - [Test] - public void TestSetEquality() - { - Ast.SetNode set1 = new Ast.SetNode(); - set1.Elements.Add(new Ast.IntegerNode(1)); - set1.Elements.Add(new Ast.IntegerNode(2)); - set1.Elements.Add(new Ast.IntegerNode(3)); - - Ast.SetNode set2 = new Ast.SetNode(); - set2.Elements.Add(new Ast.IntegerNode(2)); - set2.Elements.Add(new Ast.IntegerNode(3)); - set2.Elements.Add(new Ast.IntegerNode(1)); - - Assert.AreEqual(set1, set2); - set2.Elements.Add(new Ast.IntegerNode(0)); - Assert.AreNotEqual(set1, set2); - } - - [Test] - public void TestPrintSingle() - { - Parser p = new Parser(); - - // primitives - Assert.AreEqual("42", p.Parse("42").Root.ToString()); - Assert.AreEqual("-42.331", p.Parse("-42.331").Root.ToString()); - Assert.AreEqual("-42.0", p.Parse("-42.0").Root.ToString()); - Assert.AreEqual("-2E+20", p.Parse("-2E20").Root.ToString()); - Assert.AreEqual("2.0", p.Parse("2.0").Root.ToString()); - Assert.AreEqual("1.2E+19", p.Parse("1.2e19").Root.ToString()); - Assert.AreEqual("True", p.Parse("True").Root.ToString()); - Assert.AreEqual("'hello'", p.Parse("'hello'").Root.ToString()); - Assert.AreEqual("'\\n'", p.Parse("'\n'").Root.ToString()); - Assert.AreEqual("'\\''", p.Parse("'\\''").Root.ToString()); - Assert.AreEqual("'\"'", p.Parse("'\\\"'").Root.ToString()); - Assert.AreEqual("'\"'", p.Parse("'\"'").Root.ToString()); - Assert.AreEqual("'\\\\'", p.Parse("'\\\\'").Root.ToString()); - Assert.AreEqual("None", p.Parse("None").Root.ToString()); - string ustr = "'\u20ac\u2603'"; - Assert.AreEqual(ustr, p.Parse(ustr).Root.ToString()); - - // complex - Assert.AreEqual("(0+2j)", p.Parse("2j").Root.ToString()); - Assert.AreEqual("(-1.1-2.2j)", p.Parse("(-1.1-2.2j)").Root.ToString()); - Assert.AreEqual("(1.1+2.2j)", p.Parse("(1.1+2.2j)").Root.ToString()); - - // long int - Assert.AreEqual("123456789123456789123456789", p.Parse("123456789123456789123456789").Root.ToString()); - } - - [Test] - public void TestPrintSeq() - { - Parser p=new Parser(); - - //tuple - Assert.AreEqual("()", p.Parse("()").Root.ToString()); - Assert.AreEqual("(42,)", p.Parse("(42,)").Root.ToString()); - Assert.AreEqual("(42,43)", p.Parse("(42,43)").Root.ToString()); - - // list - Assert.AreEqual("[]", p.Parse("[]").Root.ToString()); - Assert.AreEqual("[42]", p.Parse("[42]").Root.ToString()); - Assert.AreEqual("[42,43]", p.Parse("[42,43]").Root.ToString()); - - // set - Assert.AreEqual("{42}", p.Parse("{42}").Root.ToString()); - Assert.AreEqual("{42,43}", p.Parse("{42,43,43,43}").Root.ToString()); - - // dict - Assert.AreEqual("{}", p.Parse("{}").Root.ToString()); - Assert.AreEqual("{'a':42}", p.Parse("{'a': 42}").Root.ToString()); - Assert.AreEqual("{'a':42,'b':43}", p.Parse("{'a': 42, 'b': 43}").Root.ToString()); - Assert.AreEqual("{'a':42,'b':45}", p.Parse("{'a': 42, 'b': 43, 'b': 44, 'b': 45}").Root.ToString()); - } - - [Test] - public void TestInvalidPrimitives() - { - Parser p = new Parser(); - Assert.Throws(()=>p.Parse("1+2")); - Assert.Throws(()=>p.Parse("1-2")); - Assert.Throws(()=>p.Parse("1.1+2.2")); - Assert.Throws(()=>p.Parse("1.1-2.2")); - Assert.Throws(()=>p.Parse("True+2")); - Assert.Throws(()=>p.Parse("False-2")); - Assert.Throws(()=>p.Parse("3j+2")); - Assert.Throws(()=>p.Parse("3j-2")); - Assert.Throws(()=>p.Parse("None+2")); - Assert.Throws(()=>p.Parse("None-2")); - } - - [Test] - public void TestComplex() - { - Parser p = new Parser(); - var cplx = new Ast.ComplexNumberNode() { - Real = 4.2, - Imaginary = 3.2 - }; - var cplx2 = new Ast.ComplexNumberNode() { - Real = 4.2, - Imaginary = 99 - }; - Assert.AreNotEqual(cplx, cplx2); - cplx2.Imaginary = 3.2; - Assert.AreEqual(cplx, cplx2); - - Assert.AreEqual(cplx, p.Parse("(4.2+3.2j)").Root); - cplx.Real = 0; - Assert.AreEqual(cplx, p.Parse("(0+3.2j)").Root); - Assert.AreEqual(cplx, p.Parse("3.2j").Root); - Assert.AreEqual(cplx, p.Parse("+3.2j").Root); - cplx.Imaginary = -3.2; - Assert.AreEqual(cplx, p.Parse("-3.2j").Root); - cplx.Real = -9.9; - Assert.AreEqual(cplx, p.Parse("(-9.9-3.2j)").Root); - - cplx.Real = 2; - cplx.Imaginary = 3; - Assert.AreEqual(cplx, p.Parse("(2+3j)").Root); - cplx.Imaginary = -3; - Assert.AreEqual(cplx, p.Parse("(2-3j)").Root); - cplx.Real = 0; - Assert.AreEqual(cplx, p.Parse("-3j").Root); - - cplx.Real = -3.2e32; - cplx.Imaginary = -9.9e44; - Assert.AreEqual(cplx, p.Parse("(-3.2e32 -9.9e44j)").Root); - Assert.AreEqual(cplx, p.Parse("(-3.2e+32 -9.9e+44j)").Root); - Assert.AreEqual(cplx, p.Parse("(-3.2e32-9.9e44j)").Root); - Assert.AreEqual(cplx, p.Parse("(-3.2e+32-9.9e+44j)").Root); - cplx.Imaginary = 9.9e44; - Assert.AreEqual(cplx, p.Parse("(-3.2e32+9.9e44j)").Root); - Assert.AreEqual(cplx, p.Parse("(-3.2e+32+9.9e+44j)").Root); - cplx.Real = -3.2e-32; - cplx.Imaginary = -9.9e-44; - Assert.AreEqual(cplx, p.Parse("(-3.2e-32-9.9e-44j)").Root); - } - - [Test] - public void TestComplexPrecision() - { - Parser p = new Parser(); - Ast.ComplexNumberNode cv = (Ast.ComplexNumberNode)p.Parse("(98765432123456.12345678987656+665544332211.9998877665544j)").Root; - Assert.AreEqual(98765432123456.12345678987656, cv.Real); - Assert.AreEqual(665544332211.9998877665544, cv.Imaginary); - cv = (Ast.ComplexNumberNode)p.Parse("(98765432123456.12345678987656-665544332211.9998877665544j)").Root; - Assert.AreEqual(98765432123456.12345678987656, cv.Real); - Assert.AreEqual(-665544332211.9998877665544, cv.Imaginary); - cv = (Ast.ComplexNumberNode)p.Parse("(98765432123456.12345678987656e+33+665544332211.9998877665544e+44j)").Root; - Assert.AreEqual(98765432123456.12345678987656e+33, cv.Real); - Assert.AreEqual(665544332211.9998877665544e+44, cv.Imaginary); - cv = (Ast.ComplexNumberNode)p.Parse("(-98765432123456.12345678987656e+33-665544332211.9998877665544e+44j)").Root; - Assert.AreEqual(-98765432123456.12345678987656e+33, cv.Real); - Assert.AreEqual(-665544332211.9998877665544e+44, cv.Imaginary); - } - - [Test] - public void TestPrimitivesStuffAtEnd() - { - Parser p = new Parser(); - Assert.AreEqual(new Ast.IntegerNode(42), p.ParseSingle(new SeekableStringReader("42@"))); - Assert.AreEqual(new Ast.DoubleNode(42.331), p.ParseSingle(new SeekableStringReader("42.331@"))); - Assert.AreEqual(new Ast.BooleanNode(true), p.ParseSingle(new SeekableStringReader("True@"))); - Assert.AreEqual(Ast.NoneNode.Instance, p.ParseSingle(new SeekableStringReader("None@"))); - var cplx = new Ast.ComplexNumberNode() { - Real = 4, - Imaginary = 3 - }; - Assert.AreEqual(cplx, p.ParseSingle(new SeekableStringReader("(4+3j)@"))); - cplx.Real=0; - Assert.AreEqual(cplx, p.ParseSingle(new SeekableStringReader("3j@"))); - } - - [Test] - public void TestStrings() - { - Parser p = new Parser(); - Assert.AreEqual(new Ast.StringNode("hello"), p.Parse("'hello'").Root); - Assert.AreEqual(new Ast.StringNode("hello"), p.Parse("\"hello\"").Root); - Assert.AreEqual(new Ast.StringNode("\\"), p.Parse("'\\\\'").Root); - Assert.AreEqual(new Ast.StringNode("\\"), p.Parse("\"\\\\\"").Root); - Assert.AreEqual(new Ast.StringNode("'"), p.Parse("\"'\"").Root); - Assert.AreEqual(new Ast.StringNode("\""), p.Parse("'\"'").Root); - Assert.AreEqual(new Ast.StringNode("tab\tnewline\n."), p.Parse("'tab\\tnewline\\n.'").Root); - } - - [Test] - public void TestUnicode() - { - Parser p = new Parser(); - string str = "'\u20ac\u2603'"; - Assert.AreEqual(0x20ac, str[1]); - Assert.AreEqual(0x2603, str[2]); - byte[] bytes = Encoding.UTF8.GetBytes(str); - - string value = "\u20ac\u2603"; - Assert.AreEqual(new Ast.StringNode(value), p.Parse(str).Root); - Assert.AreEqual(new Ast.StringNode(value), p.Parse(bytes).Root); - } - - [Test] - public void TestLongUnicodeRoundtrip() - { - Char[] chars64k = new Char[65536]; - for(int i=0; i<=65535; ++i) - chars64k[i]=(Char)i; - - String str64k= new String(chars64k); - - Serializer ser=new Serializer(); - byte[] data = ser.Serialize(str64k); - Assert.Greater(data.Length, chars64k.Length); - - Parser p=new Parser(); - String result = (String)p.Parse(data).GetData(); - Assert.AreEqual(str64k, result); - } - - [Test] - public void TestWhitespace() - { - Parser p = new Parser(); - Assert.AreEqual(new Ast.IntegerNode(42), p.Parse(" 42 ").Root); - Assert.AreEqual(new Ast.IntegerNode(42), p.Parse(" 42 ").Root); - Assert.AreEqual(new Ast.IntegerNode(42), p.Parse("\t42\r\n").Root); - Assert.AreEqual(new Ast.IntegerNode(42), p.Parse(" \t 42 \r \n ").Root); - Assert.AreEqual(new Ast.StringNode(" string value "), p.Parse(" ' string value ' ").Root); - Assert.Throws(()=>p.Parse(" ( 42 , ( 'x', 'y' ) ")); // missing tuple close ) - Ast ast = p.Parse(" ( 42 , ( 'x', 'y' ) ) "); - Ast.TupleNode tuple = (Ast.TupleNode) ast.Root; - Assert.AreEqual(new Ast.IntegerNode(42), tuple.Elements[0]); - tuple = (Ast.TupleNode) tuple.Elements[1]; - Assert.AreEqual(new Ast.StringNode("x"), tuple.Elements[0]); - Assert.AreEqual(new Ast.StringNode("y"), tuple.Elements[1]); - - p.Parse(" ( 52 , ) "); - p.Parse(" [ 52 ] "); - p.Parse(" { 'a' : 42 } "); - p.Parse(" { 52 } "); - } - - [Test] - public void TestTuple() - { - Parser p = new Parser(); - Ast.TupleNode tuple = new Ast.TupleNode(); - Ast.TupleNode tuple2 = new Ast.TupleNode(); - Assert.AreEqual(tuple, tuple2); - - tuple.Elements.Add(new Ast.IntegerNode(42)); - tuple2.Elements.Add(new Ast.IntegerNode(99)); - Assert.AreNotEqual(tuple, tuple2); - tuple2.Elements.Clear(); - tuple2.Elements.Add(new Ast.IntegerNode(42)); - Assert.AreEqual(tuple, tuple2); - tuple2.Elements.Add(new Ast.IntegerNode(43)); - tuple2.Elements.Add(new Ast.IntegerNode(44)); - Assert.AreNotEqual(tuple, tuple2); - - Assert.AreEqual(new Ast.TupleNode(), p.Parse("()").Root); - Assert.AreEqual(tuple, p.Parse("(42,)").Root); - Assert.AreEqual(tuple2, p.Parse("( 42,43, 44 )").Root); - - Assert.Throws(()=>p.Parse("(42,43]")); - Assert.Throws(()=>p.Parse("()@")); - Assert.Throws(()=>p.Parse("(42,43)@")); - } - - [Test] - public void TestList() - { - Parser p = new Parser(); - Ast.ListNode list = new Ast.ListNode(); - Ast.ListNode list2 = new Ast.ListNode(); - Assert.AreEqual(list, list2); - - list.Elements.Add(new Ast.IntegerNode(42)); - list2.Elements.Add(new Ast.IntegerNode(99)); - Assert.AreNotEqual(list, list2); - list2.Elements.Clear(); - list2.Elements.Add(new Ast.IntegerNode(42)); - Assert.AreEqual(list, list2); - list2.Elements.Add(new Ast.IntegerNode(43)); - list2.Elements.Add(new Ast.IntegerNode(44)); - Assert.AreNotEqual(list, list2); - - Assert.AreEqual(new Ast.ListNode(), p.Parse("[]").Root); - Assert.AreEqual(list, p.Parse("[42]").Root); - Assert.AreEqual(list2, p.Parse("[ 42,43, 44 ]").Root); - - Assert.Throws(()=>p.Parse("[42,43}")); - Assert.Throws(()=>p.Parse("[]@")); - Assert.Throws(()=>p.Parse("[42,43]@")); - } - - [Test] - public void TestSet() - { - Parser p = new Parser(); - Ast.SetNode set1 = new Ast.SetNode(); - Ast.SetNode set2 = new Ast.SetNode(); - Assert.AreEqual(set1, set2); - - set1.Elements.Add(new Ast.IntegerNode(42)); - set2.Elements.Add(new Ast.IntegerNode(99)); - Assert.AreNotEqual(set1, set2); - set2.Elements.Clear(); - set2.Elements.Add(new Ast.IntegerNode(42)); - Assert.AreEqual(set1, set2); - - set2.Elements.Add(new Ast.IntegerNode(43)); - set2.Elements.Add(new Ast.IntegerNode(44)); - Assert.AreNotEqual(set1, set2); - - Assert.AreEqual(set1, p.Parse("{42}").Root); - Assert.AreEqual(set2, p.Parse("{ 42,43, 44 }").Root); - - Assert.Throws(()=>p.Parse("{42,43]")); - Assert.Throws(()=>p.Parse("{42,43}@")); - - set1 = p.Parse("{'first','second','third','fourth','fifth','second', 'first', 'third', 'third' }").Root as Ast.SetNode; - Assert.AreEqual("'first'", set1.Elements[0].ToString()); - Assert.AreEqual("'second'", set1.Elements[1].ToString()); - Assert.AreEqual("'third'", set1.Elements[2].ToString()); - Assert.AreEqual("'fourth'", set1.Elements[3].ToString()); - Assert.AreEqual("'fifth'", set1.Elements[4].ToString()); - Assert.AreEqual(5, set1.Elements.Count); - } - - [Test] - public void TestDict() - { - Parser p = new Parser(); - Ast.DictNode dict1 = new Ast.DictNode(); - Ast.DictNode dict2 = new Ast.DictNode(); - Assert.AreEqual(dict1, dict2); - - Ast.KeyValueNode kv1 = new Ast.KeyValueNode { Key=new Ast.StringNode("key"), Value=new Ast.IntegerNode(42) }; - Ast.KeyValueNode kv2 = new Ast.KeyValueNode { Key=new Ast.StringNode("key"), Value=new Ast.IntegerNode(99) }; - Assert.AreNotEqual(kv1, kv2); - kv2.Value = new Ast.IntegerNode(42); - Assert.AreEqual(kv1, kv2); - - dict1.Elements.Add(new Ast.KeyValueNode { Key=new Ast.StringNode("key1"), Value=new Ast.IntegerNode(42) }); - dict2.Elements.Add(new Ast.KeyValueNode { Key=new Ast.StringNode("key1"), Value=new Ast.IntegerNode(99) }); - Assert.AreNotEqual(dict1, dict2); - dict2.Elements.Clear(); - dict2.Elements.Add(new Ast.KeyValueNode { Key=new Ast.StringNode("key1"), Value=new Ast.IntegerNode(42) }); - Assert.AreEqual(dict1, dict2); - - dict2.Elements.Add(new Ast.KeyValueNode { Key=new Ast.StringNode("key2"), Value=new Ast.IntegerNode(43) }); - dict2.Elements.Add(new Ast.KeyValueNode { Key=new Ast.StringNode("key3"), Value=new Ast.IntegerNode(44) }); - Assert.AreNotEqual(dict1, dict2); - - Assert.AreEqual(new Ast.DictNode(), p.Parse("{}").Root); - Assert.AreEqual(dict1, p.Parse("{'key1': 42}").Root); - Assert.AreEqual(dict2, p.Parse("{'key1': 42, 'key2': 43, 'key3':44}").Root); - - Assert.Throws(()=>p.Parse("{'key': 42]")); - Assert.Throws(()=>p.Parse("{}@")); - Assert.Throws(()=>p.Parse("{'key': 42}@")); - - dict1 = p.Parse("{'a': 1, 'b': 2, 'c': 3, 'c': 4, 'c': 5, 'c': 6}").Root as Ast.DictNode; - Assert.AreEqual("'a':1", dict1.Elements[0].ToString()); - Assert.AreEqual("'b':2", dict1.Elements[1].ToString()); - Assert.AreEqual("'c':6", dict1.Elements[2].ToString()); - Assert.AreEqual(3, dict1.Elements.Count); - } - - [Test] - public void TestFile() - { - Parser p = new Parser(); - byte[] ser=File.ReadAllBytes("testserpent.utf8.bin"); - Ast ast = p.Parse(ser); - - string expr = ast.ToString(); - Ast ast2 = p.Parse(expr); - string expr2 = ast2.ToString(); - Assert.AreEqual(expr, expr2); - - StringBuilder sb= new StringBuilder(); - Walk(ast.Root, sb); - string walk1 = sb.ToString(); - sb= new StringBuilder(); - Walk(ast2.Root, sb); - string walk2 = sb.ToString(); - Assert.AreEqual(walk1, walk2); - - // @TODO Assert.AreEqual(ast.Root, ast2.Root); - ast = p.Parse(expr2); - // @TODO Assert.AreEqual(ast.Root, ast2.Root); - } - - [Test] - [Ignore("can't yet get the ast to compare equal on mono")] - public void TestAstEquals() - { - Parser p = new Parser (); - byte[] ser = File.ReadAllBytes ("testserpent.utf8.bin"); - Ast ast = p.Parse(ser); - Ast ast2 = p.Parse(ser); - Assert.AreEqual(ast.Root, ast2.Root); - } - - public void Walk(Ast.INode node, StringBuilder sb) - { - if(node is Ast.SequenceNode) - { - sb.AppendLine(string.Format("{0} (seq)", node.GetType())); - Ast.SequenceNode seq = (Ast.SequenceNode)node; - foreach(Ast.INode child in seq.Elements) { - Walk(child, sb); - } - } - else - sb.AppendLine(string.Format("{0} = {1}", node.GetType(), node.ToString())); - } - - [Test] - public void TestTrailingCommas() - { - Parser p = new Parser(); - Ast.INode result; - result = p.Parse("[1,2,3, ]").Root; - result = p.Parse("[1,2,3 , ]").Root; - result = p.Parse("[1,2,3,]").Root; - Assert.AreEqual("[1,2,3]", result.ToString()); - result = p.Parse("(1,2,3, )").Root; - result = p.Parse("(1,2,3 , )").Root; - result = p.Parse("(1,2,3,)").Root; - Assert.AreEqual("(1,2,3)", result.ToString()); - - // for dict and set the asserts are a bit more complex - // we cannot simply convert to string because the order of elts is undefined. - - result = p.Parse("{'a':1, 'b':2, 'c':3, }").Root; - result = p.Parse("{'a':1, 'b':2, 'c':3 , }").Root; - result = p.Parse("{'a':1, 'b':2, 'c':3,}").Root; - Ast.DictNode dict = (Ast.DictNode) result; - var items = dict.ElementsAsSet(); - Assert.IsTrue(items.Contains(new Ast.KeyValueNode(new Ast.StringNode("a"), new Ast.IntegerNode(1)))); - Assert.IsTrue(items.Contains(new Ast.KeyValueNode(new Ast.StringNode("b"), new Ast.IntegerNode(2)))); - Assert.IsTrue(items.Contains(new Ast.KeyValueNode(new Ast.StringNode("c"), new Ast.IntegerNode(3)))); - result = p.Parse("{1,2,3, }").Root; - result = p.Parse("{1,2,3 , }").Root; - result = p.Parse("{1,2,3,}").Root; - Ast.SetNode set = (Ast.SetNode) result; - items = set.ElementsAsSet(); - Assert.IsTrue(items.Contains(new Ast.IntegerNode(1))); - Assert.IsTrue(items.Contains(new Ast.IntegerNode(2))); - Assert.IsTrue(items.Contains(new Ast.IntegerNode(3))); - Assert.IsFalse(items.Contains(new Ast.IntegerNode(4))); - } - } - - [TestFixture] - public class VisitorTest - { - [Test] - public void TestObjectify() - { - Parser p = new Parser(); - byte[] ser=File.ReadAllBytes("testserpent.utf8.bin"); - Ast ast = p.Parse(ser); - var visitor = new ObjectifyVisitor(); - ast.Accept(visitor); - object thing = visitor.GetObject(); - - IDictionary dict = (IDictionary) thing; - Assert.AreEqual(11, dict.Count); - IList list = dict["numbers"] as IList; - Assert.AreEqual(4, list.Count); - Assert.AreEqual(999.1234, list[1]); - Assert.AreEqual(new ComplexNumber(-3, 8), list[3]); - string euro = dict["unicode"] as string; - Assert.AreEqual("\u20ac", euro); - IDictionary exc = (IDictionary)dict["exc"]; - object[] args = (object[]) exc["args"]; - Assert.AreEqual("fault", args[0]); - Assert.AreEqual("ZeroDivisionError", exc["__class__"]); - } - - object ZerodivisionFromDict(IDictionary dict) - { - string classname = (string)dict["__class__"]; - if(classname=="ZeroDivisionError") - { - object[] args = (object[]) dict["args"]; - return new DivideByZeroException((string)args[0]); - } - return null; - } - - [Test] - public void TestObjectifyDictToClass() - { - Parser p = new Parser(); - byte[] ser=File.ReadAllBytes("testserpent.utf8.bin"); - Ast ast = p.Parse(ser); - - var visitor = new ObjectifyVisitor(ZerodivisionFromDict); - ast.Accept(visitor); - object thing = visitor.GetObject(); - - IDictionary dict = (IDictionary) thing; - Assert.AreEqual(11, dict.Count); - DivideByZeroException ex = (DivideByZeroException) dict["exc"]; - Assert.AreEqual("fault", ex.Message); - - thing = ast.GetData(ZerodivisionFromDict); - dict = (IDictionary) thing; - Assert.AreEqual(11, dict.Count); - ex = (DivideByZeroException) dict["exc"]; - Assert.AreEqual("fault", ex.Message); - } - } +/// +/// Serpent, a Python literal expression serializer/deserializer +/// (a.k.a. Python's ast.literal_eval in .NET) +/// +/// Copyright Irmen de Jong (irmen@razorvine.net) +/// Software license: "MIT software license". See http://opensource.org/licenses/MIT +/// + +using System; +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Text; +using NUnit.Framework; + +namespace Razorvine.Serpent.Test +{ + [TestFixture] + public class ParserTest + { + [Test] + public void TestBasic() + { + Parser p = new Parser(); + Assert.IsNull(p.Parse((string)null).Root); + Assert.IsNull(p.Parse("").Root); + Assert.IsNotNull(p.Parse("# comment\n42\n").Root); + } + + [Test] + public void TestComments() + { + Parser p = new Parser(); + + Ast ast = p.Parse("[ 1, 2 ]"); // no header whatsoever + var visitor = new ObjectifyVisitor(); + ast.Accept(visitor); + Object obj = visitor.GetObject(); + Assert.AreEqual(new int[] {1,2}, obj); + + ast = p.Parse(@"# serpent utf-8 python2.7 +[ 1, 2, + # some comments here + 3, 4] # more here +# and here. +"); + visitor = new ObjectifyVisitor(); + ast.Accept(visitor); + obj = visitor.GetObject(); + Assert.AreEqual(new int[] {1,2,3,4}, obj); + } + + [Test] + public void TestPrimitives() + { + Parser p = new Parser(); + Assert.AreEqual(new Ast.IntegerNode(42), p.Parse("42").Root); + Assert.AreEqual(new Ast.IntegerNode(-42), p.Parse("-42").Root); + Assert.AreEqual(new Ast.DoubleNode(42.331), p.Parse("42.331").Root); + Assert.AreEqual(new Ast.DoubleNode(-42.331), p.Parse("-42.331").Root); + Assert.AreEqual(new Ast.DoubleNode(-1.2e19), p.Parse("-1.2e+19").Root); + Assert.AreEqual(new Ast.DoubleNode(-1.2e19), p.Parse("-1.2e19").Root); + Assert.AreEqual(new Ast.DoubleNode(0.0004), p.Parse("4e-4").Root); + Assert.AreEqual(new Ast.DoubleNode(40000), p.Parse("4e4").Root); + Assert.AreEqual(new Ast.BooleanNode(true), p.Parse("True").Root); + Assert.AreEqual(new Ast.BooleanNode(false), p.Parse("False").Root); + Assert.AreEqual(Ast.NoneNode.Instance, p.Parse("None").Root); + + // long ints + Assert.AreEqual(new Ast.DecimalNode(123456789123456789123456789M), p.Parse("123456789123456789123456789").Root); + Assert.AreNotEqual(new Ast.LongNode(52), p.Parse("52").Root); + Assert.AreEqual(new Ast.LongNode(123456789123456789L), p.Parse("123456789123456789").Root); + Assert.Throws(()=>p.Parse("123456789123456789123456789123456789")); // overflow + } + + [Test] + public void TestWeirdFloats() + { + Parser p = new Parser(); + var d = (Ast.DoubleNode) p.Parse("1e30000").Root; + Assert.IsTrue(double.IsPositiveInfinity(d.Value)); + d = (Ast.DoubleNode) p.Parse("-1e30000").Root; + Assert.IsTrue(double.IsNegativeInfinity(d.Value)); + + var tuple = (Ast.TupleNode) p.Parse("(1e30000,-1e30000,{'__class__':'float','value':'nan'})").Root; + Assert.AreEqual(3, tuple.Elements.Count); + d = (Ast.DoubleNode) tuple.Elements[0]; + Assert.IsTrue(double.IsPositiveInfinity(d.Value)); + d = (Ast.DoubleNode) tuple.Elements[1]; + Assert.IsTrue(double.IsNegativeInfinity(d.Value)); + d = (Ast.DoubleNode) tuple.Elements[2]; + Assert.IsTrue(Double.IsNaN(d.Value)); + + var c = (Ast.ComplexNumberNode) p.Parse("(1e30000-1e30000j)").Root; + Assert.IsTrue(double.IsPositiveInfinity(c.Real)); + Assert.IsTrue(double.IsNegativeInfinity(c.Imaginary)); + } + + [Test] + public void TestFloatPrecision() + { + Parser p = new Parser(); + Serializer serpent = new Serializer(); + var ser = serpent.Serialize(1.2345678987654321); + Ast.DoubleNode dv = (Ast.DoubleNode) p.Parse(ser).Root; + Assert.AreEqual(1.2345678987654321.ToString(), dv.Value.ToString()); + + ser = serpent.Serialize(5555.12345678987656); + dv = (Ast.DoubleNode) p.Parse(ser).Root; + Assert.AreEqual(5555.12345678987656.ToString(), dv.Value.ToString()); + + ser = serpent.Serialize(98765432123456.12345678987656); + dv = (Ast.DoubleNode) p.Parse(ser).Root; + Assert.AreEqual(98765432123456.12345678987656.ToString(), dv.Value.ToString()); + + ser = serpent.Serialize(98765432123456.12345678987656e+44); + dv = (Ast.DoubleNode) p.Parse(ser).Root; + Assert.AreEqual((98765432123456.12345678987656e+44).ToString(), dv.Value.ToString()); + + ser = serpent.Serialize(-98765432123456.12345678987656e-44); + dv = (Ast.DoubleNode) p.Parse(ser).Root; + Assert.AreEqual((-98765432123456.12345678987656e-44).ToString(), dv.Value.ToString()); + } + + [Test] + public void TestEquality() + { + Ast.INode n1, n2; + + n1 = new Ast.IntegerNode(42); + n2 = new Ast.IntegerNode(42); + Assert.AreEqual(n1, n2); + n2 = new Ast.IntegerNode(43); + Assert.AreNotEqual(n1, n2); + + n1 = new Ast.StringNode("foo"); + n2 = new Ast.StringNode("foo"); + Assert.AreEqual(n1, n2); + n2 = new Ast.StringNode("bar"); + Assert.AreNotEqual(n1, n2); + + n1 = new Ast.ComplexNumberNode() { + Real=1.1, + Imaginary=2.2 + }; + n2 = new Ast.ComplexNumberNode() { + Real=1.1, + Imaginary=2.2 + }; + Assert.AreEqual(n1, n2); + n2 = new Ast.ComplexNumberNode() { + Real=1.1, + Imaginary=3.3 + }; + Assert.AreNotEqual(n1, n2); + + n1=new Ast.KeyValueNode() { + Key=new Ast.IntegerNode(42), + Value=new Ast.IntegerNode(42) + }; + n2=new Ast.KeyValueNode() { + Key=new Ast.IntegerNode(42), + Value=new Ast.IntegerNode(42) + }; + Assert.AreEqual(n1, n2); + n1=new Ast.KeyValueNode() { + Key=new Ast.IntegerNode(43), + Value=new Ast.IntegerNode(43) + }; + Assert.AreNotEqual(n1,n2); + + n1=Ast.NoneNode.Instance; + n2=Ast.NoneNode.Instance; + Assert.AreEqual(n1, n2); + n2=new Ast.IntegerNode(42); + Assert.AreNotEqual(n1, n2); + + n1=new Ast.DictNode() { + Elements=new List() { + new Ast.KeyValueNode() { + Key=new Ast.IntegerNode(42), + Value=new Ast.IntegerNode(42) + } + } + }; + n2=new Ast.DictNode() { + Elements=new List() { + new Ast.KeyValueNode() { + Key=new Ast.IntegerNode(42), + Value=new Ast.IntegerNode(42) + } + } + }; + Assert.AreEqual(n1, n2); + n2=new Ast.DictNode() { + Elements=new List() { + new Ast.KeyValueNode() { + Key=new Ast.IntegerNode(42), + Value=new Ast.IntegerNode(43) + } + } + }; + Assert.AreNotEqual(n1, n2); + + n1=new Ast.ListNode() { + Elements=new List() { + new Ast.IntegerNode(42) + } + }; + n2=new Ast.ListNode() { + Elements=new List() { + new Ast.IntegerNode(42) + } + }; + Assert.AreEqual(n1,n2); + n2=new Ast.ListNode() { + Elements=new List() { + new Ast.IntegerNode(43) + } + }; + Assert.AreNotEqual(n1,n2); + + n1=new Ast.SetNode() { + Elements=new List() { + new Ast.IntegerNode(42) + } + }; + n2=new Ast.SetNode() { + Elements=new List() { + new Ast.IntegerNode(42) + } + }; + Assert.AreEqual(n1,n2); + n2=new Ast.SetNode() { + Elements=new List() { + new Ast.IntegerNode(43) + } + }; + Assert.AreNotEqual(n1,n2); + + n1=new Ast.TupleNode() { + Elements=new List() { + new Ast.IntegerNode(42) + } + }; + n2=new Ast.TupleNode() { + Elements=new List() { + new Ast.IntegerNode(42) + } + }; + Assert.AreEqual(n1,n2); + n2=new Ast.TupleNode() { + Elements=new List() { + new Ast.IntegerNode(43) + } + }; + Assert.AreNotEqual(n1,n2); + + } + + [Test] + public void TestDictEquality() + { + Ast.DictNode dict1 = new Ast.DictNode(); + Ast.KeyValueNode kv=new Ast.KeyValueNode() { + Key=new Ast.StringNode("key1"), + Value=new Ast.IntegerNode(42) + }; + dict1.Elements.Add(kv); + kv=new Ast.KeyValueNode() { + Key=new Ast.StringNode("key2"), + Value=new Ast.IntegerNode(43) + }; + dict1.Elements.Add(kv); + kv=new Ast.KeyValueNode() { + Key=new Ast.StringNode("key3"), + Value=new Ast.IntegerNode(44) + }; + dict1.Elements.Add(kv); + + Ast.DictNode dict2 = new Ast.DictNode(); + kv=new Ast.KeyValueNode(){ + Key=new Ast.StringNode("key2"), + Value=new Ast.IntegerNode(43) + }; + dict2.Elements.Add(kv); + kv=new Ast.KeyValueNode() { + Key=new Ast.StringNode("key3"), + Value=new Ast.IntegerNode(44) + }; + dict2.Elements.Add(kv); + kv=new Ast.KeyValueNode() { + Key=new Ast.StringNode("key1"), + Value=new Ast.IntegerNode(42) + }; + dict2.Elements.Add(kv); + + Assert.AreEqual(dict1, dict2); + kv=new Ast.KeyValueNode() { + Key=new Ast.StringNode("key4"), + Value=new Ast.IntegerNode(45) + }; + dict2.Elements.Add(kv); + Assert.AreNotEqual(dict1, dict2); + } + + [Test] + public void TestSetEquality() + { + Ast.SetNode set1 = new Ast.SetNode(); + set1.Elements.Add(new Ast.IntegerNode(1)); + set1.Elements.Add(new Ast.IntegerNode(2)); + set1.Elements.Add(new Ast.IntegerNode(3)); + + Ast.SetNode set2 = new Ast.SetNode(); + set2.Elements.Add(new Ast.IntegerNode(2)); + set2.Elements.Add(new Ast.IntegerNode(3)); + set2.Elements.Add(new Ast.IntegerNode(1)); + + Assert.AreEqual(set1, set2); + set2.Elements.Add(new Ast.IntegerNode(0)); + Assert.AreNotEqual(set1, set2); + } + + [Test] + public void TestPrintSingle() + { + Parser p = new Parser(); + + // primitives + Assert.AreEqual("42", p.Parse("42").Root.ToString()); + Assert.AreEqual("-42.331", p.Parse("-42.331").Root.ToString()); + Assert.AreEqual("-42.0", p.Parse("-42.0").Root.ToString()); + Assert.AreEqual("-2E+20", p.Parse("-2E20").Root.ToString()); + Assert.AreEqual("2.0", p.Parse("2.0").Root.ToString()); + Assert.AreEqual("1.2E+19", p.Parse("1.2e19").Root.ToString()); + Assert.AreEqual("True", p.Parse("True").Root.ToString()); + Assert.AreEqual("'hello'", p.Parse("'hello'").Root.ToString()); + Assert.AreEqual("'\\n'", p.Parse("'\n'").Root.ToString()); + Assert.AreEqual("'\\''", p.Parse("'\\''").Root.ToString()); + Assert.AreEqual("'\"'", p.Parse("'\\\"'").Root.ToString()); + Assert.AreEqual("'\"'", p.Parse("'\"'").Root.ToString()); + Assert.AreEqual("'\\\\'", p.Parse("'\\\\'").Root.ToString()); + Assert.AreEqual("None", p.Parse("None").Root.ToString()); + string ustr = "'\u20ac\u2603'"; + Assert.AreEqual(ustr, p.Parse(ustr).Root.ToString()); + + // complex + Assert.AreEqual("(0+2j)", p.Parse("2j").Root.ToString()); + Assert.AreEqual("(-1.1-2.2j)", p.Parse("(-1.1-2.2j)").Root.ToString()); + Assert.AreEqual("(1.1+2.2j)", p.Parse("(1.1+2.2j)").Root.ToString()); + + // long int + Assert.AreEqual("123456789123456789123456789", p.Parse("123456789123456789123456789").Root.ToString()); + } + + [Test] + public void TestPrintSeq() + { + Parser p=new Parser(); + + //tuple + Assert.AreEqual("()", p.Parse("()").Root.ToString()); + Assert.AreEqual("(42,)", p.Parse("(42,)").Root.ToString()); + Assert.AreEqual("(42,43)", p.Parse("(42,43)").Root.ToString()); + + // list + Assert.AreEqual("[]", p.Parse("[]").Root.ToString()); + Assert.AreEqual("[42]", p.Parse("[42]").Root.ToString()); + Assert.AreEqual("[42,43]", p.Parse("[42,43]").Root.ToString()); + + // set + Assert.AreEqual("{42}", p.Parse("{42}").Root.ToString()); + Assert.AreEqual("{42,43}", p.Parse("{42,43,43,43}").Root.ToString()); + + // dict + Assert.AreEqual("{}", p.Parse("{}").Root.ToString()); + Assert.AreEqual("{'a':42}", p.Parse("{'a': 42}").Root.ToString()); + Assert.AreEqual("{'a':42,'b':43}", p.Parse("{'a': 42, 'b': 43}").Root.ToString()); + Assert.AreEqual("{'a':42,'b':45}", p.Parse("{'a': 42, 'b': 43, 'b': 44, 'b': 45}").Root.ToString()); + } + + [Test] + public void TestInvalidPrimitives() + { + Parser p = new Parser(); + Assert.Throws(()=>p.Parse("1+2")); + Assert.Throws(()=>p.Parse("1-2")); + Assert.Throws(()=>p.Parse("1.1+2.2")); + Assert.Throws(()=>p.Parse("1.1-2.2")); + Assert.Throws(()=>p.Parse("True+2")); + Assert.Throws(()=>p.Parse("False-2")); + Assert.Throws(()=>p.Parse("3j+2")); + Assert.Throws(()=>p.Parse("3j-2")); + Assert.Throws(()=>p.Parse("None+2")); + Assert.Throws(()=>p.Parse("None-2")); + } + + [Test] + public void TestComplex() + { + Parser p = new Parser(); + var cplx = new Ast.ComplexNumberNode() { + Real = 4.2, + Imaginary = 3.2 + }; + var cplx2 = new Ast.ComplexNumberNode() { + Real = 4.2, + Imaginary = 99 + }; + Assert.AreNotEqual(cplx, cplx2); + cplx2.Imaginary = 3.2; + Assert.AreEqual(cplx, cplx2); + + Assert.AreEqual(cplx, p.Parse("(4.2+3.2j)").Root); + cplx.Real = 0; + Assert.AreEqual(cplx, p.Parse("(0+3.2j)").Root); + Assert.AreEqual(cplx, p.Parse("3.2j").Root); + Assert.AreEqual(cplx, p.Parse("+3.2j").Root); + cplx.Imaginary = -3.2; + Assert.AreEqual(cplx, p.Parse("-3.2j").Root); + cplx.Real = -9.9; + Assert.AreEqual(cplx, p.Parse("(-9.9-3.2j)").Root); + + cplx.Real = 2; + cplx.Imaginary = 3; + Assert.AreEqual(cplx, p.Parse("(2+3j)").Root); + cplx.Imaginary = -3; + Assert.AreEqual(cplx, p.Parse("(2-3j)").Root); + cplx.Real = 0; + Assert.AreEqual(cplx, p.Parse("-3j").Root); + + cplx.Real = -3.2e32; + cplx.Imaginary = -9.9e44; + Assert.AreEqual(cplx, p.Parse("(-3.2e32 -9.9e44j)").Root); + Assert.AreEqual(cplx, p.Parse("(-3.2e+32 -9.9e+44j)").Root); + Assert.AreEqual(cplx, p.Parse("(-3.2e32-9.9e44j)").Root); + Assert.AreEqual(cplx, p.Parse("(-3.2e+32-9.9e+44j)").Root); + cplx.Imaginary = 9.9e44; + Assert.AreEqual(cplx, p.Parse("(-3.2e32+9.9e44j)").Root); + Assert.AreEqual(cplx, p.Parse("(-3.2e+32+9.9e+44j)").Root); + cplx.Real = -3.2e-32; + cplx.Imaginary = -9.9e-44; + Assert.AreEqual(cplx, p.Parse("(-3.2e-32-9.9e-44j)").Root); + } + + [Test] + public void TestComplexPrecision() + { + Parser p = new Parser(); + Ast.ComplexNumberNode cv = (Ast.ComplexNumberNode)p.Parse("(98765432123456.12345678987656+665544332211.9998877665544j)").Root; + Assert.AreEqual(98765432123456.12345678987656, cv.Real); + Assert.AreEqual(665544332211.9998877665544, cv.Imaginary); + cv = (Ast.ComplexNumberNode)p.Parse("(98765432123456.12345678987656-665544332211.9998877665544j)").Root; + Assert.AreEqual(98765432123456.12345678987656, cv.Real); + Assert.AreEqual(-665544332211.9998877665544, cv.Imaginary); + cv = (Ast.ComplexNumberNode)p.Parse("(98765432123456.12345678987656e+33+665544332211.9998877665544e+44j)").Root; + Assert.AreEqual(98765432123456.12345678987656e+33, cv.Real); + Assert.AreEqual(665544332211.9998877665544e+44, cv.Imaginary); + cv = (Ast.ComplexNumberNode)p.Parse("(-98765432123456.12345678987656e+33-665544332211.9998877665544e+44j)").Root; + Assert.AreEqual(-98765432123456.12345678987656e+33, cv.Real); + Assert.AreEqual(-665544332211.9998877665544e+44, cv.Imaginary); + } + + [Test] + public void TestPrimitivesStuffAtEnd() + { + Parser p = new Parser(); + Assert.AreEqual(new Ast.IntegerNode(42), p.ParseSingle(new SeekableStringReader("42@"))); + Assert.AreEqual(new Ast.DoubleNode(42.331), p.ParseSingle(new SeekableStringReader("42.331@"))); + Assert.AreEqual(new Ast.BooleanNode(true), p.ParseSingle(new SeekableStringReader("True@"))); + Assert.AreEqual(Ast.NoneNode.Instance, p.ParseSingle(new SeekableStringReader("None@"))); + var cplx = new Ast.ComplexNumberNode() { + Real = 4, + Imaginary = 3 + }; + Assert.AreEqual(cplx, p.ParseSingle(new SeekableStringReader("(4+3j)@"))); + cplx.Real=0; + Assert.AreEqual(cplx, p.ParseSingle(new SeekableStringReader("3j@"))); + } + + [Test] + public void TestStrings() + { + Parser p = new Parser(); + Assert.AreEqual(new Ast.StringNode("hello"), p.Parse("'hello'").Root); + Assert.AreEqual(new Ast.StringNode("hello"), p.Parse("\"hello\"").Root); + Assert.AreEqual(new Ast.StringNode("\\"), p.Parse("'\\\\'").Root); + Assert.AreEqual(new Ast.StringNode("\\"), p.Parse("\"\\\\\"").Root); + Assert.AreEqual(new Ast.StringNode("'"), p.Parse("\"'\"").Root); + Assert.AreEqual(new Ast.StringNode("\""), p.Parse("'\"'").Root); + Assert.AreEqual(new Ast.StringNode("tab\tnewline\n."), p.Parse("'tab\\tnewline\\n.'").Root); + } + + [Test] + public void TestUnicode() + { + Parser p = new Parser(); + string str = "'\u20ac\u2603'"; + Assert.AreEqual(0x20ac, str[1]); + Assert.AreEqual(0x2603, str[2]); + byte[] bytes = Encoding.UTF8.GetBytes(str); + + string value = "\u20ac\u2603"; + Assert.AreEqual(new Ast.StringNode(value), p.Parse(str).Root); + Assert.AreEqual(new Ast.StringNode(value), p.Parse(bytes).Root); + } + + [Test] + public void TestLongUnicodeRoundtrip() + { + Char[] chars64k = new Char[65536]; + for(int i=0; i<=65535; ++i) + chars64k[i]=(Char)i; + + String str64k= new String(chars64k); + + Serializer ser=new Serializer(); + byte[] data = ser.Serialize(str64k); + Assert.Greater(data.Length, chars64k.Length); + + Parser p=new Parser(); + String result = (String)p.Parse(data).GetData(); + Assert.AreEqual(str64k, result); + } + + [Test] + public void TestWhitespace() + { + Parser p = new Parser(); + Assert.AreEqual(new Ast.IntegerNode(42), p.Parse(" 42 ").Root); + Assert.AreEqual(new Ast.IntegerNode(42), p.Parse(" 42 ").Root); + Assert.AreEqual(new Ast.IntegerNode(42), p.Parse("\t42\r\n").Root); + Assert.AreEqual(new Ast.IntegerNode(42), p.Parse(" \t 42 \r \n ").Root); + Assert.AreEqual(new Ast.StringNode(" string value "), p.Parse(" ' string value ' ").Root); + Assert.Throws(()=>p.Parse(" ( 42 , ( 'x', 'y' ) ")); // missing tuple close ) + Ast ast = p.Parse(" ( 42 , ( 'x', 'y' ) ) "); + Ast.TupleNode tuple = (Ast.TupleNode) ast.Root; + Assert.AreEqual(new Ast.IntegerNode(42), tuple.Elements[0]); + tuple = (Ast.TupleNode) tuple.Elements[1]; + Assert.AreEqual(new Ast.StringNode("x"), tuple.Elements[0]); + Assert.AreEqual(new Ast.StringNode("y"), tuple.Elements[1]); + + p.Parse(" ( 52 , ) "); + p.Parse(" [ 52 ] "); + p.Parse(" { 'a' : 42 } "); + p.Parse(" { 52 } "); + } + + [Test] + public void TestTuple() + { + Parser p = new Parser(); + Ast.TupleNode tuple = new Ast.TupleNode(); + Ast.TupleNode tuple2 = new Ast.TupleNode(); + Assert.AreEqual(tuple, tuple2); + + tuple.Elements.Add(new Ast.IntegerNode(42)); + tuple2.Elements.Add(new Ast.IntegerNode(99)); + Assert.AreNotEqual(tuple, tuple2); + tuple2.Elements.Clear(); + tuple2.Elements.Add(new Ast.IntegerNode(42)); + Assert.AreEqual(tuple, tuple2); + tuple2.Elements.Add(new Ast.IntegerNode(43)); + tuple2.Elements.Add(new Ast.IntegerNode(44)); + Assert.AreNotEqual(tuple, tuple2); + + Assert.AreEqual(new Ast.TupleNode(), p.Parse("()").Root); + Assert.AreEqual(tuple, p.Parse("(42,)").Root); + Assert.AreEqual(tuple2, p.Parse("( 42,43, 44 )").Root); + + Assert.Throws(()=>p.Parse("(42,43]")); + Assert.Throws(()=>p.Parse("()@")); + Assert.Throws(()=>p.Parse("(42,43)@")); + } + + [Test] + public void TestList() + { + Parser p = new Parser(); + Ast.ListNode list = new Ast.ListNode(); + Ast.ListNode list2 = new Ast.ListNode(); + Assert.AreEqual(list, list2); + + list.Elements.Add(new Ast.IntegerNode(42)); + list2.Elements.Add(new Ast.IntegerNode(99)); + Assert.AreNotEqual(list, list2); + list2.Elements.Clear(); + list2.Elements.Add(new Ast.IntegerNode(42)); + Assert.AreEqual(list, list2); + list2.Elements.Add(new Ast.IntegerNode(43)); + list2.Elements.Add(new Ast.IntegerNode(44)); + Assert.AreNotEqual(list, list2); + + Assert.AreEqual(new Ast.ListNode(), p.Parse("[]").Root); + Assert.AreEqual(list, p.Parse("[42]").Root); + Assert.AreEqual(list2, p.Parse("[ 42,43, 44 ]").Root); + + Assert.Throws(()=>p.Parse("[42,43}")); + Assert.Throws(()=>p.Parse("[]@")); + Assert.Throws(()=>p.Parse("[42,43]@")); + } + + [Test] + public void TestSet() + { + Parser p = new Parser(); + Ast.SetNode set1 = new Ast.SetNode(); + Ast.SetNode set2 = new Ast.SetNode(); + Assert.AreEqual(set1, set2); + + set1.Elements.Add(new Ast.IntegerNode(42)); + set2.Elements.Add(new Ast.IntegerNode(99)); + Assert.AreNotEqual(set1, set2); + set2.Elements.Clear(); + set2.Elements.Add(new Ast.IntegerNode(42)); + Assert.AreEqual(set1, set2); + + set2.Elements.Add(new Ast.IntegerNode(43)); + set2.Elements.Add(new Ast.IntegerNode(44)); + Assert.AreNotEqual(set1, set2); + + Assert.AreEqual(set1, p.Parse("{42}").Root); + Assert.AreEqual(set2, p.Parse("{ 42,43, 44 }").Root); + + Assert.Throws(()=>p.Parse("{42,43]")); + Assert.Throws(()=>p.Parse("{42,43}@")); + + set1 = p.Parse("{'first','second','third','fourth','fifth','second', 'first', 'third', 'third' }").Root as Ast.SetNode; + Assert.AreEqual("'first'", set1.Elements[0].ToString()); + Assert.AreEqual("'second'", set1.Elements[1].ToString()); + Assert.AreEqual("'third'", set1.Elements[2].ToString()); + Assert.AreEqual("'fourth'", set1.Elements[3].ToString()); + Assert.AreEqual("'fifth'", set1.Elements[4].ToString()); + Assert.AreEqual(5, set1.Elements.Count); + } + + [Test] + public void TestDict() + { + Parser p = new Parser(); + Ast.DictNode dict1 = new Ast.DictNode(); + Ast.DictNode dict2 = new Ast.DictNode(); + Assert.AreEqual(dict1, dict2); + + Ast.KeyValueNode kv1 = new Ast.KeyValueNode { Key=new Ast.StringNode("key"), Value=new Ast.IntegerNode(42) }; + Ast.KeyValueNode kv2 = new Ast.KeyValueNode { Key=new Ast.StringNode("key"), Value=new Ast.IntegerNode(99) }; + Assert.AreNotEqual(kv1, kv2); + kv2.Value = new Ast.IntegerNode(42); + Assert.AreEqual(kv1, kv2); + + dict1.Elements.Add(new Ast.KeyValueNode { Key=new Ast.StringNode("key1"), Value=new Ast.IntegerNode(42) }); + dict2.Elements.Add(new Ast.KeyValueNode { Key=new Ast.StringNode("key1"), Value=new Ast.IntegerNode(99) }); + Assert.AreNotEqual(dict1, dict2); + dict2.Elements.Clear(); + dict2.Elements.Add(new Ast.KeyValueNode { Key=new Ast.StringNode("key1"), Value=new Ast.IntegerNode(42) }); + Assert.AreEqual(dict1, dict2); + + dict2.Elements.Add(new Ast.KeyValueNode { Key=new Ast.StringNode("key2"), Value=new Ast.IntegerNode(43) }); + dict2.Elements.Add(new Ast.KeyValueNode { Key=new Ast.StringNode("key3"), Value=new Ast.IntegerNode(44) }); + Assert.AreNotEqual(dict1, dict2); + + Assert.AreEqual(new Ast.DictNode(), p.Parse("{}").Root); + Assert.AreEqual(dict1, p.Parse("{'key1': 42}").Root); + Assert.AreEqual(dict2, p.Parse("{'key1': 42, 'key2': 43, 'key3':44}").Root); + + Assert.Throws(()=>p.Parse("{'key': 42]")); + Assert.Throws(()=>p.Parse("{}@")); + Assert.Throws(()=>p.Parse("{'key': 42}@")); + + dict1 = p.Parse("{'a': 1, 'b': 2, 'c': 3, 'c': 4, 'c': 5, 'c': 6}").Root as Ast.DictNode; + Assert.AreEqual("'a':1", dict1.Elements[0].ToString()); + Assert.AreEqual("'b':2", dict1.Elements[1].ToString()); + Assert.AreEqual("'c':6", dict1.Elements[2].ToString()); + Assert.AreEqual(3, dict1.Elements.Count); + } + + [Test] + public void TestFile() + { + Parser p = new Parser(); + byte[] ser=File.ReadAllBytes("testserpent.utf8.bin"); + Ast ast = p.Parse(ser); + + string expr = ast.ToString(); + Ast ast2 = p.Parse(expr); + string expr2 = ast2.ToString(); + Assert.AreEqual(expr, expr2); + + StringBuilder sb= new StringBuilder(); + Walk(ast.Root, sb); + string walk1 = sb.ToString(); + sb= new StringBuilder(); + Walk(ast2.Root, sb); + string walk2 = sb.ToString(); + Assert.AreEqual(walk1, walk2); + + // @TODO Assert.AreEqual(ast.Root, ast2.Root); + ast = p.Parse(expr2); + // @TODO Assert.AreEqual(ast.Root, ast2.Root); + } + + [Test] + [Ignore("can't yet get the ast to compare equal on mono")] + public void TestAstEquals() + { + Parser p = new Parser (); + byte[] ser = File.ReadAllBytes ("testserpent.utf8.bin"); + Ast ast = p.Parse(ser); + Ast ast2 = p.Parse(ser); + Assert.AreEqual(ast.Root, ast2.Root); + } + + public void Walk(Ast.INode node, StringBuilder sb) + { + if(node is Ast.SequenceNode) + { + sb.AppendLine(string.Format("{0} (seq)", node.GetType())); + Ast.SequenceNode seq = (Ast.SequenceNode)node; + foreach(Ast.INode child in seq.Elements) { + Walk(child, sb); + } + } + else + sb.AppendLine(string.Format("{0} = {1}", node.GetType(), node.ToString())); + } + + [Test] + public void TestTrailingCommas() + { + Parser p = new Parser(); + Ast.INode result; + result = p.Parse("[1,2,3, ]").Root; + result = p.Parse("[1,2,3 , ]").Root; + result = p.Parse("[1,2,3,]").Root; + Assert.AreEqual("[1,2,3]", result.ToString()); + result = p.Parse("(1,2,3, )").Root; + result = p.Parse("(1,2,3 , )").Root; + result = p.Parse("(1,2,3,)").Root; + Assert.AreEqual("(1,2,3)", result.ToString()); + + // for dict and set the asserts are a bit more complex + // we cannot simply convert to string because the order of elts is undefined. + + result = p.Parse("{'a':1, 'b':2, 'c':3, }").Root; + result = p.Parse("{'a':1, 'b':2, 'c':3 , }").Root; + result = p.Parse("{'a':1, 'b':2, 'c':3,}").Root; + Ast.DictNode dict = (Ast.DictNode) result; + var items = dict.ElementsAsSet(); + Assert.IsTrue(items.Contains(new Ast.KeyValueNode(new Ast.StringNode("a"), new Ast.IntegerNode(1)))); + Assert.IsTrue(items.Contains(new Ast.KeyValueNode(new Ast.StringNode("b"), new Ast.IntegerNode(2)))); + Assert.IsTrue(items.Contains(new Ast.KeyValueNode(new Ast.StringNode("c"), new Ast.IntegerNode(3)))); + result = p.Parse("{1,2,3, }").Root; + result = p.Parse("{1,2,3 , }").Root; + result = p.Parse("{1,2,3,}").Root; + Ast.SetNode set = (Ast.SetNode) result; + items = set.ElementsAsSet(); + Assert.IsTrue(items.Contains(new Ast.IntegerNode(1))); + Assert.IsTrue(items.Contains(new Ast.IntegerNode(2))); + Assert.IsTrue(items.Contains(new Ast.IntegerNode(3))); + Assert.IsFalse(items.Contains(new Ast.IntegerNode(4))); + } + } + + [TestFixture] + public class VisitorTest + { + [Test] + public void TestObjectify() + { + Parser p = new Parser(); + byte[] ser=File.ReadAllBytes("testserpent.utf8.bin"); + Ast ast = p.Parse(ser); + var visitor = new ObjectifyVisitor(); + ast.Accept(visitor); + object thing = visitor.GetObject(); + + IDictionary dict = (IDictionary) thing; + Assert.AreEqual(11, dict.Count); + IList list = dict["numbers"] as IList; + Assert.AreEqual(4, list.Count); + Assert.AreEqual(999.1234, list[1]); + Assert.AreEqual(new ComplexNumber(-3, 8), list[3]); + string euro = dict["unicode"] as string; + Assert.AreEqual("\u20ac", euro); + IDictionary exc = (IDictionary)dict["exc"]; + object[] args = (object[]) exc["args"]; + Assert.AreEqual("fault", args[0]); + Assert.AreEqual("ZeroDivisionError", exc["__class__"]); + } + + object ZerodivisionFromDict(IDictionary dict) + { + string classname = (string)dict["__class__"]; + if(classname=="ZeroDivisionError") + { + object[] args = (object[]) dict["args"]; + return new DivideByZeroException((string)args[0]); + } + return null; + } + + [Test] + public void TestObjectifyDictToClass() + { + Parser p = new Parser(); + byte[] ser=File.ReadAllBytes("testserpent.utf8.bin"); + Ast ast = p.Parse(ser); + + var visitor = new ObjectifyVisitor(ZerodivisionFromDict); + ast.Accept(visitor); + object thing = visitor.GetObject(); + + IDictionary dict = (IDictionary) thing; + Assert.AreEqual(11, dict.Count); + DivideByZeroException ex = (DivideByZeroException) dict["exc"]; + Assert.AreEqual("fault", ex.Message); + + thing = ast.GetData(ZerodivisionFromDict); + dict = (IDictionary) thing; + Assert.AreEqual(11, dict.Count); + ex = (DivideByZeroException) dict["exc"]; + Assert.AreEqual("fault", ex.Message); + } + } } \ No newline at end of file diff --git a/dotnet/Serpent.Test/SerializeTest.cs b/dotnet/Serpent.Test/SerializeTest.cs index 367538d..bf0346e 100644 --- a/dotnet/Serpent.Test/SerializeTest.cs +++ b/dotnet/Serpent.Test/SerializeTest.cs @@ -1,743 +1,743 @@ -/// -/// Serpent, a Python literal expression serializer/deserializer -/// (a.k.a. Python's ast.literal_eval in .NET) -/// -/// Copyright Irmen de Jong (irmen@razorvine.net) -/// Software license: "MIT software license". See http://opensource.org/licenses/MIT -/// - -using System; -using System.Collections.Generic; -using System.Text; - -using NUnit.Framework; -using Hashtable = System.Collections.Hashtable; -using IDictionary = System.Collections.IDictionary; - -namespace Razorvine.Serpent.Test -{ - [TestFixture] - public class SerializeTest - { - public byte[] strip_header(byte[] data) - { - int start=Array.IndexOf(data, (byte)10); // the newline after the header - if(start<0) - throw new ArgumentException("need header in string"); - start++; - byte[] result = new byte[data.Length-start]; - Array.Copy(data, start, result, 0, data.Length-start); - return result; - } - - public byte[] B(string s) - { - return Encoding.UTF8.GetBytes(s); - } - - public string S(byte[] b) - { - return Encoding.UTF8.GetString(b); - } - - - [Test] - public void TestHeader() - { - Serializer ser = new Serializer(); - byte[] data = ser.Serialize(null); - Assert.AreEqual(35, data[0]); - string strdata = S(data); - Assert.AreEqual("# serpent utf-8 python3.2", strdata.Split('\n')[0]); - - ser.SetLiterals=false; - data = ser.Serialize(null); - strdata = S(data); - Assert.AreEqual("# serpent utf-8 python2.6", strdata.Split('\n')[0]); - - data = B("# header\nfirst-line"); - data = strip_header(data); - Assert.AreEqual(B("first-line"), data); - } - - - [Test] - public void TestStuff() - { - Serializer ser=new Serializer(); - byte[] result = ser.Serialize("blerp"); - result=strip_header(result); - Assert.AreEqual(B("'blerp'"), result); - result = ser.Serialize(new Guid("f1f8d00e-49a5-4662-ac1d-d5f0426ed293")); - result=strip_header(result); - Assert.AreEqual(B("'f1f8d00e-49a5-4662-ac1d-d5f0426ed293'"), result); - result = ser.Serialize(123456789.987654321987654321987654321987654321m); - result=strip_header(result); - Assert.AreEqual(B("'123456789.98765432198765432199'"), result); - } - - [Test] - public void TestNull() - { - Serializer ser = new Serializer(); - byte[] data = ser.Serialize(null); - data=strip_header(data); - Assert.AreEqual(B("None"),data); - } - - [Test] - public void TestStrings() - { - Serializer serpent = new Serializer(); - byte[] ser = serpent.Serialize("hello"); - byte[] data = strip_header(ser); - Assert.AreEqual(B("'hello'"), data); - ser = serpent.Serialize("quotes'\""); - data = strip_header(ser); - Assert.AreEqual(B("'quotes\\'\"'"), data); - ser = serpent.Serialize("quotes2'"); - data = strip_header(ser); - Assert.AreEqual(B("\"quotes2'\""), data); - } - - [Test] - public void TestUnicodeEscapes() - { - Serializer serpent=new Serializer(); - - // regular escaped chars first - byte[] ser = serpent.Serialize("\b\r\n\f\t \\"); - byte[] data = strip_header(ser); - // '\\x08\\r\\n\\x0c\\t \\\\' - Assert.AreEqual(new byte[] {39, - 92, 120, 48, 56, - 92, 114, - 92, 110, - 92, 120, 48, 99, - 92, 116, - 32, - 92, 92, - 39}, data); - - // simple cases (chars < 0x80) - ser = serpent.Serialize("\u0000\u0001\u001f\u007f"); - data = strip_header(ser); - // '\\x00\\x01\\x1f\\x7f' - Assert.AreEqual(new byte[] {39, - 92, 120, 48, 48, - 92, 120, 48, 49, - 92, 120, 49, 102, - 92, 120, 55, 102, - 39 }, data); - - // chars 0x80 .. 0xff - ser = serpent.Serialize("\u0080\u0081\u00ff"); - data = strip_header(ser); - // '\\x80\\x81\xc3\xbf' (has some utf-8 encoded chars in it) - Assert.AreEqual(new byte[] {39, - 92, 120, 56, 48, - 92, 120, 56, 49, - 195, 191, - 39}, data); - - // chars above 0xff - ser = serpent.Serialize("\u0100\u20ac\u8899"); - data = strip_header(ser); - // '\xc4\x80\xe2\x82\xac\xe8\xa2\x99' (has some utf-8 encoded chars in it) - Assert.AreEqual(new byte[] {39, 196, 128, 226, 130, 172, 232, 162, 153, 39}, data); - -// // some random high chars that are all printable in python and not escaped -// ser = serpent.Serialize("\u0377\u082d\u10c5\u135d\uac00"); -// data = strip_header(ser); -// Console.WriteLine(S(data)); // XXX -// // '\xcd\xb7\xe0\xa0\xad\xe1\x83\x85\xe1\x8d\x9d\xea\xb0\x80' (only a bunch of utf-8 encoded chars) -// Assert.AreEqual(new byte[] {39, 205, 183, 224, 160, 173, 225, 131, 133, 225, 141, 157, 234, 176, 128, 39}, data); - - // some random high chars that are all non-printable in python and that are escaped - ser = serpent.Serialize("\u0378\u082e\u10c6\u135c\uabff"); - data = strip_header(ser); - // '\\u0378\\u082e\\u10c6\\u135c\\uabff' - Assert.AreEqual(new byte[] {39, - 92, 117, 48, 51, 55, 56, - 92, 117, 48, 56, 50, 101, - 92, 117, 49, 48, 99, 54, - 92, 117, 49, 51, 53, 99, - 92, 117, 97, 98, 102, 102, - 39}, data); - } - - [Test] - public void TestNumbers() - { - Serializer serpent = new Serializer(); - byte[] ser = serpent.Serialize((int)12345); - byte[] data = strip_header(ser); - Assert.AreEqual(B("12345"), data); - ser = serpent.Serialize((uint)12345); - data = strip_header(ser); - Assert.AreEqual(B("12345"), data); - ser = serpent.Serialize((long)1234567891234567891L); - data = strip_header(ser); - Assert.AreEqual(B("1234567891234567891"), data); - ser = serpent.Serialize((ulong)12345678912345678912L); - data = strip_header(ser); - Assert.AreEqual(B("12345678912345678912"), data); - ser = serpent.Serialize(99.1234); - data = strip_header(ser); - Assert.AreEqual(B("99.1234"), data); - ser = serpent.Serialize(1234.9999999999m); - data = strip_header(ser); - Assert.AreEqual(B("'1234.9999999999'"), data); - ser = serpent.Serialize(123456789.987654321987654321987654321987654321m); - data=strip_header(ser); - Assert.AreEqual(B("'123456789.98765432198765432199'"), data); - ComplexNumber cplx = new ComplexNumber(2.2, 3.3); - ser = serpent.Serialize(cplx); - data = strip_header(ser); - Assert.AreEqual(B("(2.2+3.3j)"), data); - cplx = new ComplexNumber(0, 3); - ser = serpent.Serialize(cplx); - data = strip_header(ser); - Assert.AreEqual(B("(0+3j)"), data); - cplx = new ComplexNumber(-2, -3); - ser = serpent.Serialize(cplx); - data = strip_header(ser); - Assert.AreEqual(B("(-2-3j)"), data); - } - - [Test] - public void TestDoubleNanInf() - { - Serializer serpent = new Serializer(); - var doubles = new object[] {double.PositiveInfinity, double.NegativeInfinity, double.NaN, - float.PositiveInfinity, float.NegativeInfinity, float.NaN, - new ComplexNumber(double.PositiveInfinity, 3.4)}; - byte[] ser = serpent.Serialize(doubles); - byte[] data = strip_header(ser); - Assert.AreEqual("(1e30000,-1e30000,{'__class__':'float','value':'nan'},1e30000,-1e30000,{'__class__':'float','value':'nan'},(1e30000+3.4j))", S(data)); - } - - [Test] - public void TestBool() - { - Serializer serpent = new Serializer(); - byte[] ser = serpent.Serialize(true); - byte[] data = strip_header(ser); - Assert.AreEqual(B("True"),data); - ser = serpent.Serialize(false); - data = strip_header(ser); - Assert.AreEqual(B("False"),data); - } - - [Test] - public void TestList() - { - Serializer serpent = new Serializer(); - IList list = new List(); - - // test empty list - byte[] ser = strip_header(serpent.Serialize(list)); - Assert.AreEqual("[]", S(ser)); - serpent.Indent=true; - ser = strip_header(serpent.Serialize(list)); - Assert.AreEqual("[]", S(ser)); - serpent.Indent=false; - - // test nonempty list - list.Add(42); - list.Add("Sally"); - list.Add(16.5); - ser = strip_header(serpent.Serialize(list)); - Assert.AreEqual("[42,'Sally',16.5]", S(ser)); - serpent.Indent=true; - ser = strip_header(serpent.Serialize(list)); - Assert.AreEqual("[\n 42,\n 'Sally',\n 16.5\n]", S(ser)); - } - - [Test] - public void TestSet() - { - // test with set literals - Serializer serpent = new Serializer(); - serpent.SetLiterals = true; - HashSet set = new HashSet(); - - // test empty set - byte[] ser = strip_header(serpent.Serialize(set)); - Assert.AreEqual("()", S(ser)); // empty set is serialized as a tuple. - serpent.Indent=true; - ser = strip_header(serpent.Serialize(set)); - Assert.AreEqual("()", S(ser)); // empty set is serialized as a tuple. - serpent.Indent=false; - - // test nonempty set - set.Add(42); - set.Add("Sally"); - set.Add(16.5); - ser = strip_header(serpent.Serialize(set)); - Assert.AreEqual("{42,'Sally',16.5}", S(ser)); - serpent.Indent=true; - ser = strip_header(serpent.Serialize(set)); - Assert.AreEqual("{\n 42,\n 'Sally',\n 16.5\n}", S(ser)); - - // test no set literals - serpent.Indent=false; - serpent.SetLiterals=false; - ser = strip_header(serpent.Serialize(set)); - Assert.AreEqual("(42,'Sally',16.5)", S(ser)); // needs to be tuple now - } - - [Test] - public void TestDictionary() - { - Serializer serpent = new Serializer(); - Parser p = new Parser(); - - // test empty dict - IDictionary ht = new Hashtable(); - byte[] ser = serpent.Serialize(ht); - Assert.AreEqual(B("{}"), strip_header(ser)); - string parsed = p.Parse(ser).Root.ToString(); - Assert.AreEqual("{}", parsed); - - // empty dict with indentation - serpent.Indent=true; - ser = serpent.Serialize(ht); - Assert.AreEqual(B("{}"), strip_header(ser)); - parsed = p.Parse(ser).Root.ToString(); - Assert.AreEqual("{}", parsed); - - // test dict with values - serpent.Indent=false; - ht = new Hashtable() { - {42, "fortytwo"}, - {"sixteen-and-half", 16.5}, - {"name", "Sally"}, - {"status", false} - }; - - ser = serpent.Serialize(ht); - Assert.AreEqual('}', ser[ser.Length-1]); - Assert.AreNotEqual(',', ser[ser.Length-2]); - parsed = p.Parse(ser).Root.ToString(); - Assert.AreEqual(69, parsed.Length); - - // test indentation - serpent.Indent=true; - ser = serpent.Serialize(ht); - Assert.AreEqual('}', ser[ser.Length-1]); - Assert.AreEqual('\n', ser[ser.Length-2]); - Assert.AreNotEqual(',', ser[ser.Length-3]); - string ser_str = S(strip_header(ser)); - Assert.IsTrue(ser_str.Contains("'name': 'Sally'")); - Assert.IsTrue(ser_str.Contains("'status': False")); - Assert.IsTrue(ser_str.Contains("42: 'fortytwo'")); - Assert.IsTrue(ser_str.Contains("'sixteen-and-half': 16.5")); - parsed = p.Parse(ser).Root.ToString(); - Assert.AreEqual(69, parsed.Length); - serpent.Indent=false; - - // generic Dictionary test - IDictionary mydict = new Dictionary { - { 1, "one" }, - { 2, "two" }, - }; - ser = serpent.Serialize(mydict); - ser_str = S(strip_header(ser)); - Assert.IsTrue(ser_str=="{2:'two',1:'one'}" || ser_str=="{1:'one',2:'two'}"); - } - - [Test] - public void TestBytes() - { - Serializer serpent = new Serializer(indent: true); - byte[] bytes = new byte[] { 97, 98, 99, 100, 101, 102 }; // abcdef - byte[] ser = serpent.Serialize(bytes); - Assert.AreEqual("{\n 'data': 'YWJjZGVm',\n 'encoding': 'base64'\n}", S(strip_header(ser))); - - Parser p = new Parser(); - string parsed = p.Parse(ser).Root.ToString(); - Assert.AreEqual(39, parsed.Length); - - var hashtable = new Hashtable { - {"data", "YWJjZGVm"}, - {"encoding", "base64"} - }; - byte[] bytes2 = Parser.ToBytes(hashtable); - Assert.AreEqual(bytes, bytes2); - - var dict = new Dictionary { - {"data", "YWJjZGVm"}, - {"encoding", "base64"} - }; - bytes2 = Parser.ToBytes(dict); - Assert.AreEqual(bytes, bytes2); - - var dict2 = new Dictionary { - {"data", "YWJjZGVm"}, - {"encoding", "base64"} - }; - bytes2 = Parser.ToBytes(dict2); - Assert.AreEqual(bytes, bytes2); - - dict["encoding"] = "base99"; - Assert.Throws(()=>Parser.ToBytes(dict)); - dict.Clear(); - Assert.Throws(()=>Parser.ToBytes(dict)); - dict.Clear(); - dict["data"] = "YWJjZGVm"; - Assert.Throws(()=>Parser.ToBytes(dict)); - dict.Clear(); - dict["encoding"] = "base64"; - Assert.Throws(()=>Parser.ToBytes(dict)); - Assert.Throws(()=>Parser.ToBytes(12345)); - Assert.Throws(()=>Parser.ToBytes(null)); - } - - [Test] - public void TestCollection() - { - ICollection intlist = new LinkedList(); - intlist.Add(42); - intlist.Add(43); - Serializer serpent = new Serializer(); - byte[] ser = serpent.Serialize(intlist); - ser = strip_header(ser); - Assert.AreEqual("[42,43]", S(ser)); - - ser=strip_header(serpent.Serialize(new int[] {42})); - Assert.AreEqual("(42,)", S(ser)); - ser=strip_header(serpent.Serialize(new int[] {42, 43})); - Assert.AreEqual("(42,43)", S(ser)); - - serpent.Indent=true; - ser = strip_header(serpent.Serialize(intlist)); - Assert.AreEqual("[\n 42,\n 43\n]", S(ser)); - ser=strip_header(serpent.Serialize(new int[] {42})); - Assert.AreEqual("(\n 42,\n)", S(ser)); - ser=strip_header(serpent.Serialize(new int[] {42, 43})); - Assert.AreEqual("(\n 42,\n 43\n)", S(ser)); - } - - - [Test] - public void TestIndentation() - { - var dict = new Dictionary(); - var list = new List() { - 1, - 2, - new string[] {"a", "b"} - }; - dict.Add("first", list); - dict.Add("second", new Dictionary { - {1, false} - }); - dict.Add("third", new HashSet { 3, 4} ); - - Serializer serpent = new Serializer(); - serpent.Indent=true; - byte[] ser = strip_header(serpent.Serialize(dict)); - string txt=@"{ - 'first': [ - 1, - 2, - ( - 'a', - 'b' - ) - ], - 'second': { - 1: False - }, - 'third': { - 3, - 4 - } -}"; - // bit of trickery to deal with Windows/Unix line ending differences - txt = txt.Replace("\n","\r\n"); - txt = txt.Replace("\r\r\n", "\r\n"); - string ser_txt = S(ser); - ser_txt = ser_txt.Replace("\n", "\r\n"); - ser_txt = ser_txt.Replace("\r\r\n", "\r\n"); - Assert.AreEqual(txt, ser_txt); - } - - [Test] - public void TestSorting() - { - Serializer serpent=new Serializer(); - object data = new List { 3, 2, 1}; - byte[] ser = strip_header(serpent.Serialize(data)); - Assert.AreEqual("[3,2,1]", S(ser)); - data = new int[] { 3,2,1 }; - ser = strip_header(serpent.Serialize(data)); - Assert.AreEqual("(3,2,1)", S(ser)); - - data = new HashSet { - 42, - "hi" - }; - serpent.Indent=true; - ser = strip_header(serpent.Serialize(data)); - Assert.IsTrue(S(ser)=="{\n 42,\n 'hi'\n}" || S(ser)=="{\n 'hi',\n 42\n}"); - - data = new Dictionary { - {5, "five"}, - {3, "three"}, - {1, "one"}, - {4, "four"}, - {2, "two"} - }; - serpent.Indent=true; - ser = strip_header(serpent.Serialize(data)); - Assert.AreEqual("{\n 1: 'one',\n 2: 'two',\n 3: 'three',\n 4: 'four',\n 5: 'five'\n}", S(ser)); - - data = new HashSet { - "x", - "y", - "z", - "c", - "b", - "a" - }; - serpent.Indent=true; - ser = strip_header(serpent.Serialize(data)); - Assert.AreEqual("{\n 'a',\n 'b',\n 'c',\n 'x',\n 'y',\n 'z'\n}", S(ser)); - } - - [Test] - public void TestClass() - { - Serializer.RegisterClass(typeof(SerializeTestClass), null); - Serializer serpent = new Serializer(indent: true); - - var obj = new SerializeTestClass() { - i = 99, - s = "hi", - x = 42 - }; - byte[] ser = strip_header(serpent.Serialize(obj)); - Assert.AreEqual("{\n '__class__': 'SerializeTestClass',\n 'i': 99,\n 'obj': None,\n 's': 'hi'\n}", S(ser)); - } - - [Test] - public void TestClass2() - { - Serializer.RegisterClass(typeof(SerializeTestClass), null); - Serializer serpent = new Serializer(indent: true, namespaceInClassName: true); - object obj = new SerializeTestClass() { - i = 99, - s = "hi", - x = 42 - }; - byte[] ser = strip_header(serpent.Serialize(obj)); - Assert.AreEqual("{\n '__class__': 'Razorvine.Serpent.Test.SerializeTestClass',\n 'i': 99,\n 'obj': None,\n 's': 'hi'\n}", S(ser)); - } - - protected IDictionary testclassConverter(object obj) - { - SerializeTestClass o = (SerializeTestClass) obj; - IDictionary result = new Hashtable(); - result["__class@__"] = o.GetType().Name+"@"; - result["i@"] = o.i; - result["s@"] = o.s; - result["x@"] = o.x; - return result; - } - - [Test] - public void TestCustomClassDict() - { - Serializer.RegisterClass(typeof(SerializeTestClass), testclassConverter); - Serializer serpent = new Serializer(indent: true); - - var obj = new SerializeTestClass() { - i = 99, - s = "hi", - x = 42 - }; - byte[] ser = strip_header(serpent.Serialize(obj)); - Assert.AreEqual("{\n '__class@__': 'SerializeTestClass@',\n 'i@': 99,\n 's@': 'hi',\n 'x@': 42\n}", S(ser)); - } - - [Test] - public void TestStruct() - { - Serializer serpent = new Serializer(indent: true); - - var obj2 = new SerializeTestStruct() { - i = 99, - s = "hi", - x = 42 - }; - byte[] ser = strip_header(serpent.Serialize(obj2)); - Assert.AreEqual("{\n '__class__': 'SerializeTestStruct',\n 'i': 99,\n 's': 'hi'\n}", S(ser)); - } - - [Test] - public void TestStruct2() - { - Serializer serpent = new Serializer(indent: true, namespaceInClassName: true); - - var obj2 = new SerializeTestStruct() { - i = 99, - s = "hi", - x = 42 - }; - byte[] ser = strip_header(serpent.Serialize(obj2)); - Assert.AreEqual("{\n '__class__': 'Razorvine.Serpent.Test.SerializeTestStruct',\n 'i': 99,\n 's': 'hi'\n}", S(ser)); - } - - [Test] - public void TestAnonymousClass() - { - Serializer serpent = new Serializer(indent: true); - Object obj = new { - Name="Harry", - Age=33, - Country="NL" - }; - - byte[] ser = strip_header(serpent.Serialize(obj)); - Assert.AreEqual("{\n 'Age': 33,\n 'Country': 'NL',\n 'Name': 'Harry'\n}", S(ser)); - } - - [Test] - public void TestDateTime() - { - Serializer serpent = new Serializer(); - - DateTime date = new DateTime(2013, 1, 20, 23, 59, 45, 999, DateTimeKind.Local); - byte[] ser = strip_header(serpent.Serialize(date)); - Assert.AreEqual("'2013-01-20T23:59:45.999'", S(ser)); - - date = new DateTime(2013, 1, 20, 23, 59, 45, 999, DateTimeKind.Utc); - ser = strip_header(serpent.Serialize(date)); - Assert.AreEqual("'2013-01-20T23:59:45.999'", S(ser)); - - date = new DateTime(2013, 1, 20, 23, 59, 45, 999, DateTimeKind.Unspecified); - ser = strip_header(serpent.Serialize(date)); - Assert.AreEqual("'2013-01-20T23:59:45.999'", S(ser)); - - date = new DateTime(2013, 1, 20, 23, 59, 45); - ser = strip_header(serpent.Serialize(date)); - Assert.AreEqual("'2013-01-20T23:59:45'", S(ser)); - - TimeSpan timespan = new TimeSpan(1, 10, 20, 30, 999); - ser = strip_header(serpent.Serialize(timespan)); - Assert.AreEqual("123630.999", S(ser)); - } - - [Test] - public void TestDateTimeOffset() - { - Serializer serpent = new Serializer(); - - DateTimeOffset date = new DateTimeOffset(2013, 1, 20, 23, 59, 45, 999, TimeSpan.FromHours(+2)); - byte[] ser = strip_header(serpent.Serialize(date)); - Assert.AreEqual("'2013-01-20T23:59:45.999+02:00'", S(ser)); - - date = new DateTimeOffset(2013, 5, 10, 13, 59, 45, TimeSpan.FromHours(+2)); - ser = strip_header(serpent.Serialize(date)); - Assert.AreEqual("'2013-05-10T13:59:45+02:00'", S(ser)); - } - - [Test] - public void TestException() - { - Exception x = new ApplicationException("errormessage"); - Serializer serpent = new Serializer(indent:true); - byte[] ser = strip_header(serpent.Serialize(x)); - Assert.AreEqual("{\n '__class__': 'ApplicationException',\n '__exception__': True,\n 'args': (\n 'errormessage',\n ),\n 'attributes': {}\n}", S(ser)); - - x.Data["custom_attribute"]=999; - ser = strip_header(serpent.Serialize(x)); - Assert.AreEqual("{\n '__class__': 'ApplicationException',\n '__exception__': True,\n 'args': (\n 'errormessage',\n ),\n 'attributes': {\n 'custom_attribute': 999\n }\n}", S(ser)); - } - - [Test] - public void TestExceptionWithNamespace() - { - Exception x = new ApplicationException("errormessage"); - Serializer serpent = new Serializer(indent:true, namespaceInClassName: true); - byte[] ser = strip_header(serpent.Serialize(x)); - Assert.AreEqual("{\n '__class__': 'System.ApplicationException',\n '__exception__': True,\n 'args': (\n 'errormessage',\n ),\n 'attributes': {}\n}", S(ser)); - } - - enum FooType { - Foobar, - Jarjar - } - - [Test] - public void TestEnum() - { - FooType e = FooType.Jarjar; - Serializer serpent = new Serializer(); - byte[] ser = strip_header(serpent.Serialize(e)); - Assert.AreEqual("'Jarjar'", S(ser)); - } - - - interface IBaseInterface {}; - interface ISubInterface : IBaseInterface {}; - class BaseClassWithInterface : IBaseInterface {}; - class SubClassWithInterface : BaseClassWithInterface, ISubInterface {}; - class BaseClass {}; - class SubClass : BaseClass {}; - abstract class AbstractBaseClass {}; - class ConcreteSubClass : AbstractBaseClass {}; - - protected IDictionary AnyClassSerializer(object arg) - { - IDictionary result = new Hashtable(); - result["(SUB)CLASS"] = arg.GetType().Name; - return result; - } - - [Test] - public void testAbstractBaseClassHierarchyPickler() - { - ConcreteSubClass c = new ConcreteSubClass(); - Serializer serpent = new Serializer(); - serpent.Serialize(c); - - Serializer.RegisterClass(typeof(AbstractBaseClass), AnyClassSerializer); - byte[] data = serpent.Serialize(c); - Assert.AreEqual("{'(SUB)CLASS':'ConcreteSubClass'}", S(strip_header(data))); - } - - [Test] - public void TestInterfaceHierarchyPickler() - { - BaseClassWithInterface b = new BaseClassWithInterface(); - SubClassWithInterface sub = new SubClassWithInterface(); - Serializer serpent = new Serializer(); - serpent.Serialize(b); - serpent.Serialize(sub); - Serializer.RegisterClass(typeof(IBaseInterface), AnyClassSerializer); - byte[] data = serpent.Serialize(b); - Assert.AreEqual("{'(SUB)CLASS':'BaseClassWithInterface'}", S(strip_header(data))); - data = serpent.Serialize(sub); - Assert.AreEqual("{'(SUB)CLASS':'SubClassWithInterface'}", S(strip_header(data))); - } - } - - public class SerializeTestClass - { - public int x; - public string s {get; set;} - public int i {get; set;} - public object obj {get; set;} - - } - - public struct SerializeTestStruct - { - public int x; - public string s {get; set;} - public int i {get; set;} - } -} +/// +/// Serpent, a Python literal expression serializer/deserializer +/// (a.k.a. Python's ast.literal_eval in .NET) +/// +/// Copyright Irmen de Jong (irmen@razorvine.net) +/// Software license: "MIT software license". See http://opensource.org/licenses/MIT +/// + +using System; +using System.Collections.Generic; +using System.Text; + +using NUnit.Framework; +using Hashtable = System.Collections.Hashtable; +using IDictionary = System.Collections.IDictionary; + +namespace Razorvine.Serpent.Test +{ + [TestFixture] + public class SerializeTest + { + public byte[] strip_header(byte[] data) + { + int start=Array.IndexOf(data, (byte)10); // the newline after the header + if(start<0) + throw new ArgumentException("need header in string"); + start++; + byte[] result = new byte[data.Length-start]; + Array.Copy(data, start, result, 0, data.Length-start); + return result; + } + + public byte[] B(string s) + { + return Encoding.UTF8.GetBytes(s); + } + + public string S(byte[] b) + { + return Encoding.UTF8.GetString(b); + } + + + [Test] + public void TestHeader() + { + Serializer ser = new Serializer(); + byte[] data = ser.Serialize(null); + Assert.AreEqual(35, data[0]); + string strdata = S(data); + Assert.AreEqual("# serpent utf-8 python3.2", strdata.Split('\n')[0]); + + ser.SetLiterals=false; + data = ser.Serialize(null); + strdata = S(data); + Assert.AreEqual("# serpent utf-8 python2.6", strdata.Split('\n')[0]); + + data = B("# header\nfirst-line"); + data = strip_header(data); + Assert.AreEqual(B("first-line"), data); + } + + + [Test] + public void TestStuff() + { + Serializer ser=new Serializer(); + byte[] result = ser.Serialize("blerp"); + result=strip_header(result); + Assert.AreEqual(B("'blerp'"), result); + result = ser.Serialize(new Guid("f1f8d00e-49a5-4662-ac1d-d5f0426ed293")); + result=strip_header(result); + Assert.AreEqual(B("'f1f8d00e-49a5-4662-ac1d-d5f0426ed293'"), result); + result = ser.Serialize(123456789.987654321987654321987654321987654321m); + result=strip_header(result); + Assert.AreEqual(B("'123456789.98765432198765432199'"), result); + } + + [Test] + public void TestNull() + { + Serializer ser = new Serializer(); + byte[] data = ser.Serialize(null); + data=strip_header(data); + Assert.AreEqual(B("None"),data); + } + + [Test] + public void TestStrings() + { + Serializer serpent = new Serializer(); + byte[] ser = serpent.Serialize("hello"); + byte[] data = strip_header(ser); + Assert.AreEqual(B("'hello'"), data); + ser = serpent.Serialize("quotes'\""); + data = strip_header(ser); + Assert.AreEqual(B("'quotes\\'\"'"), data); + ser = serpent.Serialize("quotes2'"); + data = strip_header(ser); + Assert.AreEqual(B("\"quotes2'\""), data); + } + + [Test] + public void TestUnicodeEscapes() + { + Serializer serpent=new Serializer(); + + // regular escaped chars first + byte[] ser = serpent.Serialize("\b\r\n\f\t \\"); + byte[] data = strip_header(ser); + // '\\x08\\r\\n\\x0c\\t \\\\' + Assert.AreEqual(new byte[] {39, + 92, 120, 48, 56, + 92, 114, + 92, 110, + 92, 120, 48, 99, + 92, 116, + 32, + 92, 92, + 39}, data); + + // simple cases (chars < 0x80) + ser = serpent.Serialize("\u0000\u0001\u001f\u007f"); + data = strip_header(ser); + // '\\x00\\x01\\x1f\\x7f' + Assert.AreEqual(new byte[] {39, + 92, 120, 48, 48, + 92, 120, 48, 49, + 92, 120, 49, 102, + 92, 120, 55, 102, + 39 }, data); + + // chars 0x80 .. 0xff + ser = serpent.Serialize("\u0080\u0081\u00ff"); + data = strip_header(ser); + // '\\x80\\x81\xc3\xbf' (has some utf-8 encoded chars in it) + Assert.AreEqual(new byte[] {39, + 92, 120, 56, 48, + 92, 120, 56, 49, + 195, 191, + 39}, data); + + // chars above 0xff + ser = serpent.Serialize("\u0100\u20ac\u8899"); + data = strip_header(ser); + // '\xc4\x80\xe2\x82\xac\xe8\xa2\x99' (has some utf-8 encoded chars in it) + Assert.AreEqual(new byte[] {39, 196, 128, 226, 130, 172, 232, 162, 153, 39}, data); + +// // some random high chars that are all printable in python and not escaped +// ser = serpent.Serialize("\u0377\u082d\u10c5\u135d\uac00"); +// data = strip_header(ser); +// Console.WriteLine(S(data)); // XXX +// // '\xcd\xb7\xe0\xa0\xad\xe1\x83\x85\xe1\x8d\x9d\xea\xb0\x80' (only a bunch of utf-8 encoded chars) +// Assert.AreEqual(new byte[] {39, 205, 183, 224, 160, 173, 225, 131, 133, 225, 141, 157, 234, 176, 128, 39}, data); + + // some random high chars that are all non-printable in python and that are escaped + ser = serpent.Serialize("\u0378\u082e\u10c6\u135c\uabff"); + data = strip_header(ser); + // '\\u0378\\u082e\\u10c6\\u135c\\uabff' + Assert.AreEqual(new byte[] {39, + 92, 117, 48, 51, 55, 56, + 92, 117, 48, 56, 50, 101, + 92, 117, 49, 48, 99, 54, + 92, 117, 49, 51, 53, 99, + 92, 117, 97, 98, 102, 102, + 39}, data); + } + + [Test] + public void TestNumbers() + { + Serializer serpent = new Serializer(); + byte[] ser = serpent.Serialize((int)12345); + byte[] data = strip_header(ser); + Assert.AreEqual(B("12345"), data); + ser = serpent.Serialize((uint)12345); + data = strip_header(ser); + Assert.AreEqual(B("12345"), data); + ser = serpent.Serialize((long)1234567891234567891L); + data = strip_header(ser); + Assert.AreEqual(B("1234567891234567891"), data); + ser = serpent.Serialize((ulong)12345678912345678912L); + data = strip_header(ser); + Assert.AreEqual(B("12345678912345678912"), data); + ser = serpent.Serialize(99.1234); + data = strip_header(ser); + Assert.AreEqual(B("99.1234"), data); + ser = serpent.Serialize(1234.9999999999m); + data = strip_header(ser); + Assert.AreEqual(B("'1234.9999999999'"), data); + ser = serpent.Serialize(123456789.987654321987654321987654321987654321m); + data=strip_header(ser); + Assert.AreEqual(B("'123456789.98765432198765432199'"), data); + ComplexNumber cplx = new ComplexNumber(2.2, 3.3); + ser = serpent.Serialize(cplx); + data = strip_header(ser); + Assert.AreEqual(B("(2.2+3.3j)"), data); + cplx = new ComplexNumber(0, 3); + ser = serpent.Serialize(cplx); + data = strip_header(ser); + Assert.AreEqual(B("(0+3j)"), data); + cplx = new ComplexNumber(-2, -3); + ser = serpent.Serialize(cplx); + data = strip_header(ser); + Assert.AreEqual(B("(-2-3j)"), data); + } + + [Test] + public void TestDoubleNanInf() + { + Serializer serpent = new Serializer(); + var doubles = new object[] {double.PositiveInfinity, double.NegativeInfinity, double.NaN, + float.PositiveInfinity, float.NegativeInfinity, float.NaN, + new ComplexNumber(double.PositiveInfinity, 3.4)}; + byte[] ser = serpent.Serialize(doubles); + byte[] data = strip_header(ser); + Assert.AreEqual("(1e30000,-1e30000,{'__class__':'float','value':'nan'},1e30000,-1e30000,{'__class__':'float','value':'nan'},(1e30000+3.4j))", S(data)); + } + + [Test] + public void TestBool() + { + Serializer serpent = new Serializer(); + byte[] ser = serpent.Serialize(true); + byte[] data = strip_header(ser); + Assert.AreEqual(B("True"),data); + ser = serpent.Serialize(false); + data = strip_header(ser); + Assert.AreEqual(B("False"),data); + } + + [Test] + public void TestList() + { + Serializer serpent = new Serializer(); + IList list = new List(); + + // test empty list + byte[] ser = strip_header(serpent.Serialize(list)); + Assert.AreEqual("[]", S(ser)); + serpent.Indent=true; + ser = strip_header(serpent.Serialize(list)); + Assert.AreEqual("[]", S(ser)); + serpent.Indent=false; + + // test nonempty list + list.Add(42); + list.Add("Sally"); + list.Add(16.5); + ser = strip_header(serpent.Serialize(list)); + Assert.AreEqual("[42,'Sally',16.5]", S(ser)); + serpent.Indent=true; + ser = strip_header(serpent.Serialize(list)); + Assert.AreEqual("[\n 42,\n 'Sally',\n 16.5\n]", S(ser)); + } + + [Test] + public void TestSet() + { + // test with set literals + Serializer serpent = new Serializer(); + serpent.SetLiterals = true; + HashSet set = new HashSet(); + + // test empty set + byte[] ser = strip_header(serpent.Serialize(set)); + Assert.AreEqual("()", S(ser)); // empty set is serialized as a tuple. + serpent.Indent=true; + ser = strip_header(serpent.Serialize(set)); + Assert.AreEqual("()", S(ser)); // empty set is serialized as a tuple. + serpent.Indent=false; + + // test nonempty set + set.Add(42); + set.Add("Sally"); + set.Add(16.5); + ser = strip_header(serpent.Serialize(set)); + Assert.AreEqual("{42,'Sally',16.5}", S(ser)); + serpent.Indent=true; + ser = strip_header(serpent.Serialize(set)); + Assert.AreEqual("{\n 42,\n 'Sally',\n 16.5\n}", S(ser)); + + // test no set literals + serpent.Indent=false; + serpent.SetLiterals=false; + ser = strip_header(serpent.Serialize(set)); + Assert.AreEqual("(42,'Sally',16.5)", S(ser)); // needs to be tuple now + } + + [Test] + public void TestDictionary() + { + Serializer serpent = new Serializer(); + Parser p = new Parser(); + + // test empty dict + IDictionary ht = new Hashtable(); + byte[] ser = serpent.Serialize(ht); + Assert.AreEqual(B("{}"), strip_header(ser)); + string parsed = p.Parse(ser).Root.ToString(); + Assert.AreEqual("{}", parsed); + + // empty dict with indentation + serpent.Indent=true; + ser = serpent.Serialize(ht); + Assert.AreEqual(B("{}"), strip_header(ser)); + parsed = p.Parse(ser).Root.ToString(); + Assert.AreEqual("{}", parsed); + + // test dict with values + serpent.Indent=false; + ht = new Hashtable() { + {42, "fortytwo"}, + {"sixteen-and-half", 16.5}, + {"name", "Sally"}, + {"status", false} + }; + + ser = serpent.Serialize(ht); + Assert.AreEqual('}', ser[ser.Length-1]); + Assert.AreNotEqual(',', ser[ser.Length-2]); + parsed = p.Parse(ser).Root.ToString(); + Assert.AreEqual(69, parsed.Length); + + // test indentation + serpent.Indent=true; + ser = serpent.Serialize(ht); + Assert.AreEqual('}', ser[ser.Length-1]); + Assert.AreEqual('\n', ser[ser.Length-2]); + Assert.AreNotEqual(',', ser[ser.Length-3]); + string ser_str = S(strip_header(ser)); + Assert.IsTrue(ser_str.Contains("'name': 'Sally'")); + Assert.IsTrue(ser_str.Contains("'status': False")); + Assert.IsTrue(ser_str.Contains("42: 'fortytwo'")); + Assert.IsTrue(ser_str.Contains("'sixteen-and-half': 16.5")); + parsed = p.Parse(ser).Root.ToString(); + Assert.AreEqual(69, parsed.Length); + serpent.Indent=false; + + // generic Dictionary test + IDictionary mydict = new Dictionary { + { 1, "one" }, + { 2, "two" }, + }; + ser = serpent.Serialize(mydict); + ser_str = S(strip_header(ser)); + Assert.IsTrue(ser_str=="{2:'two',1:'one'}" || ser_str=="{1:'one',2:'two'}"); + } + + [Test] + public void TestBytes() + { + Serializer serpent = new Serializer(indent: true); + byte[] bytes = new byte[] { 97, 98, 99, 100, 101, 102 }; // abcdef + byte[] ser = serpent.Serialize(bytes); + Assert.AreEqual("{\n 'data': 'YWJjZGVm',\n 'encoding': 'base64'\n}", S(strip_header(ser))); + + Parser p = new Parser(); + string parsed = p.Parse(ser).Root.ToString(); + Assert.AreEqual(39, parsed.Length); + + var hashtable = new Hashtable { + {"data", "YWJjZGVm"}, + {"encoding", "base64"} + }; + byte[] bytes2 = Parser.ToBytes(hashtable); + Assert.AreEqual(bytes, bytes2); + + var dict = new Dictionary { + {"data", "YWJjZGVm"}, + {"encoding", "base64"} + }; + bytes2 = Parser.ToBytes(dict); + Assert.AreEqual(bytes, bytes2); + + var dict2 = new Dictionary { + {"data", "YWJjZGVm"}, + {"encoding", "base64"} + }; + bytes2 = Parser.ToBytes(dict2); + Assert.AreEqual(bytes, bytes2); + + dict["encoding"] = "base99"; + Assert.Throws(()=>Parser.ToBytes(dict)); + dict.Clear(); + Assert.Throws(()=>Parser.ToBytes(dict)); + dict.Clear(); + dict["data"] = "YWJjZGVm"; + Assert.Throws(()=>Parser.ToBytes(dict)); + dict.Clear(); + dict["encoding"] = "base64"; + Assert.Throws(()=>Parser.ToBytes(dict)); + Assert.Throws(()=>Parser.ToBytes(12345)); + Assert.Throws(()=>Parser.ToBytes(null)); + } + + [Test] + public void TestCollection() + { + ICollection intlist = new LinkedList(); + intlist.Add(42); + intlist.Add(43); + Serializer serpent = new Serializer(); + byte[] ser = serpent.Serialize(intlist); + ser = strip_header(ser); + Assert.AreEqual("[42,43]", S(ser)); + + ser=strip_header(serpent.Serialize(new int[] {42})); + Assert.AreEqual("(42,)", S(ser)); + ser=strip_header(serpent.Serialize(new int[] {42, 43})); + Assert.AreEqual("(42,43)", S(ser)); + + serpent.Indent=true; + ser = strip_header(serpent.Serialize(intlist)); + Assert.AreEqual("[\n 42,\n 43\n]", S(ser)); + ser=strip_header(serpent.Serialize(new int[] {42})); + Assert.AreEqual("(\n 42,\n)", S(ser)); + ser=strip_header(serpent.Serialize(new int[] {42, 43})); + Assert.AreEqual("(\n 42,\n 43\n)", S(ser)); + } + + + [Test] + public void TestIndentation() + { + var dict = new Dictionary(); + var list = new List() { + 1, + 2, + new string[] {"a", "b"} + }; + dict.Add("first", list); + dict.Add("second", new Dictionary { + {1, false} + }); + dict.Add("third", new HashSet { 3, 4} ); + + Serializer serpent = new Serializer(); + serpent.Indent=true; + byte[] ser = strip_header(serpent.Serialize(dict)); + string txt=@"{ + 'first': [ + 1, + 2, + ( + 'a', + 'b' + ) + ], + 'second': { + 1: False + }, + 'third': { + 3, + 4 + } +}"; + // bit of trickery to deal with Windows/Unix line ending differences + txt = txt.Replace("\n","\r\n"); + txt = txt.Replace("\r\r\n", "\r\n"); + string ser_txt = S(ser); + ser_txt = ser_txt.Replace("\n", "\r\n"); + ser_txt = ser_txt.Replace("\r\r\n", "\r\n"); + Assert.AreEqual(txt, ser_txt); + } + + [Test] + public void TestSorting() + { + Serializer serpent=new Serializer(); + object data = new List { 3, 2, 1}; + byte[] ser = strip_header(serpent.Serialize(data)); + Assert.AreEqual("[3,2,1]", S(ser)); + data = new int[] { 3,2,1 }; + ser = strip_header(serpent.Serialize(data)); + Assert.AreEqual("(3,2,1)", S(ser)); + + data = new HashSet { + 42, + "hi" + }; + serpent.Indent=true; + ser = strip_header(serpent.Serialize(data)); + Assert.IsTrue(S(ser)=="{\n 42,\n 'hi'\n}" || S(ser)=="{\n 'hi',\n 42\n}"); + + data = new Dictionary { + {5, "five"}, + {3, "three"}, + {1, "one"}, + {4, "four"}, + {2, "two"} + }; + serpent.Indent=true; + ser = strip_header(serpent.Serialize(data)); + Assert.AreEqual("{\n 1: 'one',\n 2: 'two',\n 3: 'three',\n 4: 'four',\n 5: 'five'\n}", S(ser)); + + data = new HashSet { + "x", + "y", + "z", + "c", + "b", + "a" + }; + serpent.Indent=true; + ser = strip_header(serpent.Serialize(data)); + Assert.AreEqual("{\n 'a',\n 'b',\n 'c',\n 'x',\n 'y',\n 'z'\n}", S(ser)); + } + + [Test] + public void TestClass() + { + Serializer.RegisterClass(typeof(SerializeTestClass), null); + Serializer serpent = new Serializer(indent: true); + + var obj = new SerializeTestClass() { + i = 99, + s = "hi", + x = 42 + }; + byte[] ser = strip_header(serpent.Serialize(obj)); + Assert.AreEqual("{\n '__class__': 'SerializeTestClass',\n 'i': 99,\n 'obj': None,\n 's': 'hi'\n}", S(ser)); + } + + [Test] + public void TestClass2() + { + Serializer.RegisterClass(typeof(SerializeTestClass), null); + Serializer serpent = new Serializer(indent: true, namespaceInClassName: true); + object obj = new SerializeTestClass() { + i = 99, + s = "hi", + x = 42 + }; + byte[] ser = strip_header(serpent.Serialize(obj)); + Assert.AreEqual("{\n '__class__': 'Razorvine.Serpent.Test.SerializeTestClass',\n 'i': 99,\n 'obj': None,\n 's': 'hi'\n}", S(ser)); + } + + protected IDictionary testclassConverter(object obj) + { + SerializeTestClass o = (SerializeTestClass) obj; + IDictionary result = new Hashtable(); + result["__class@__"] = o.GetType().Name+"@"; + result["i@"] = o.i; + result["s@"] = o.s; + result["x@"] = o.x; + return result; + } + + [Test] + public void TestCustomClassDict() + { + Serializer.RegisterClass(typeof(SerializeTestClass), testclassConverter); + Serializer serpent = new Serializer(indent: true); + + var obj = new SerializeTestClass() { + i = 99, + s = "hi", + x = 42 + }; + byte[] ser = strip_header(serpent.Serialize(obj)); + Assert.AreEqual("{\n '__class@__': 'SerializeTestClass@',\n 'i@': 99,\n 's@': 'hi',\n 'x@': 42\n}", S(ser)); + } + + [Test] + public void TestStruct() + { + Serializer serpent = new Serializer(indent: true); + + var obj2 = new SerializeTestStruct() { + i = 99, + s = "hi", + x = 42 + }; + byte[] ser = strip_header(serpent.Serialize(obj2)); + Assert.AreEqual("{\n '__class__': 'SerializeTestStruct',\n 'i': 99,\n 's': 'hi'\n}", S(ser)); + } + + [Test] + public void TestStruct2() + { + Serializer serpent = new Serializer(indent: true, namespaceInClassName: true); + + var obj2 = new SerializeTestStruct() { + i = 99, + s = "hi", + x = 42 + }; + byte[] ser = strip_header(serpent.Serialize(obj2)); + Assert.AreEqual("{\n '__class__': 'Razorvine.Serpent.Test.SerializeTestStruct',\n 'i': 99,\n 's': 'hi'\n}", S(ser)); + } + + [Test] + public void TestAnonymousClass() + { + Serializer serpent = new Serializer(indent: true); + Object obj = new { + Name="Harry", + Age=33, + Country="NL" + }; + + byte[] ser = strip_header(serpent.Serialize(obj)); + Assert.AreEqual("{\n 'Age': 33,\n 'Country': 'NL',\n 'Name': 'Harry'\n}", S(ser)); + } + + [Test] + public void TestDateTime() + { + Serializer serpent = new Serializer(); + + DateTime date = new DateTime(2013, 1, 20, 23, 59, 45, 999, DateTimeKind.Local); + byte[] ser = strip_header(serpent.Serialize(date)); + Assert.AreEqual("'2013-01-20T23:59:45.999'", S(ser)); + + date = new DateTime(2013, 1, 20, 23, 59, 45, 999, DateTimeKind.Utc); + ser = strip_header(serpent.Serialize(date)); + Assert.AreEqual("'2013-01-20T23:59:45.999'", S(ser)); + + date = new DateTime(2013, 1, 20, 23, 59, 45, 999, DateTimeKind.Unspecified); + ser = strip_header(serpent.Serialize(date)); + Assert.AreEqual("'2013-01-20T23:59:45.999'", S(ser)); + + date = new DateTime(2013, 1, 20, 23, 59, 45); + ser = strip_header(serpent.Serialize(date)); + Assert.AreEqual("'2013-01-20T23:59:45'", S(ser)); + + TimeSpan timespan = new TimeSpan(1, 10, 20, 30, 999); + ser = strip_header(serpent.Serialize(timespan)); + Assert.AreEqual("123630.999", S(ser)); + } + + [Test] + public void TestDateTimeOffset() + { + Serializer serpent = new Serializer(); + + DateTimeOffset date = new DateTimeOffset(2013, 1, 20, 23, 59, 45, 999, TimeSpan.FromHours(+2)); + byte[] ser = strip_header(serpent.Serialize(date)); + Assert.AreEqual("'2013-01-20T23:59:45.999+02:00'", S(ser)); + + date = new DateTimeOffset(2013, 5, 10, 13, 59, 45, TimeSpan.FromHours(+2)); + ser = strip_header(serpent.Serialize(date)); + Assert.AreEqual("'2013-05-10T13:59:45+02:00'", S(ser)); + } + + [Test] + public void TestException() + { + Exception x = new ApplicationException("errormessage"); + Serializer serpent = new Serializer(indent:true); + byte[] ser = strip_header(serpent.Serialize(x)); + Assert.AreEqual("{\n '__class__': 'ApplicationException',\n '__exception__': True,\n 'args': (\n 'errormessage',\n ),\n 'attributes': {}\n}", S(ser)); + + x.Data["custom_attribute"]=999; + ser = strip_header(serpent.Serialize(x)); + Assert.AreEqual("{\n '__class__': 'ApplicationException',\n '__exception__': True,\n 'args': (\n 'errormessage',\n ),\n 'attributes': {\n 'custom_attribute': 999\n }\n}", S(ser)); + } + + [Test] + public void TestExceptionWithNamespace() + { + Exception x = new ApplicationException("errormessage"); + Serializer serpent = new Serializer(indent:true, namespaceInClassName: true); + byte[] ser = strip_header(serpent.Serialize(x)); + Assert.AreEqual("{\n '__class__': 'System.ApplicationException',\n '__exception__': True,\n 'args': (\n 'errormessage',\n ),\n 'attributes': {}\n}", S(ser)); + } + + enum FooType { + Foobar, + Jarjar + } + + [Test] + public void TestEnum() + { + FooType e = FooType.Jarjar; + Serializer serpent = new Serializer(); + byte[] ser = strip_header(serpent.Serialize(e)); + Assert.AreEqual("'Jarjar'", S(ser)); + } + + + interface IBaseInterface {}; + interface ISubInterface : IBaseInterface {}; + class BaseClassWithInterface : IBaseInterface {}; + class SubClassWithInterface : BaseClassWithInterface, ISubInterface {}; + class BaseClass {}; + class SubClass : BaseClass {}; + abstract class AbstractBaseClass {}; + class ConcreteSubClass : AbstractBaseClass {}; + + protected IDictionary AnyClassSerializer(object arg) + { + IDictionary result = new Hashtable(); + result["(SUB)CLASS"] = arg.GetType().Name; + return result; + } + + [Test] + public void testAbstractBaseClassHierarchyPickler() + { + ConcreteSubClass c = new ConcreteSubClass(); + Serializer serpent = new Serializer(); + serpent.Serialize(c); + + Serializer.RegisterClass(typeof(AbstractBaseClass), AnyClassSerializer); + byte[] data = serpent.Serialize(c); + Assert.AreEqual("{'(SUB)CLASS':'ConcreteSubClass'}", S(strip_header(data))); + } + + [Test] + public void TestInterfaceHierarchyPickler() + { + BaseClassWithInterface b = new BaseClassWithInterface(); + SubClassWithInterface sub = new SubClassWithInterface(); + Serializer serpent = new Serializer(); + serpent.Serialize(b); + serpent.Serialize(sub); + Serializer.RegisterClass(typeof(IBaseInterface), AnyClassSerializer); + byte[] data = serpent.Serialize(b); + Assert.AreEqual("{'(SUB)CLASS':'BaseClassWithInterface'}", S(strip_header(data))); + data = serpent.Serialize(sub); + Assert.AreEqual("{'(SUB)CLASS':'SubClassWithInterface'}", S(strip_header(data))); + } + } + + public class SerializeTestClass + { + public int x; + public string s {get; set;} + public int i {get; set;} + public object obj {get; set;} + + } + + public struct SerializeTestStruct + { + public int x; + public string s {get; set;} + public int i {get; set;} + } +} From 0a713932c04e5d158c147bee4c687492dc6e24e6 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Fri, 17 Nov 2017 22:29:53 +0100 Subject: [PATCH 118/230] moved from Eclipse to IntelliJ IDE --- java/.classpath | 23 ------------------- java/.gitignore | 4 +++- java/.project | 23 ------------------- .../org.eclipse.core.resources.prefs | 4 ---- java/.settings/org.eclipse.jdt.core.prefs | 13 ----------- java/.settings/org.eclipse.m2e.core.prefs | 4 ---- java/serpent.iml | 17 ++++++++++++++ 7 files changed, 20 insertions(+), 68 deletions(-) delete mode 100644 java/.classpath delete mode 100644 java/.project delete mode 100644 java/.settings/org.eclipse.core.resources.prefs delete mode 100644 java/.settings/org.eclipse.jdt.core.prefs delete mode 100644 java/.settings/org.eclipse.m2e.core.prefs create mode 100644 java/serpent.iml diff --git a/java/.classpath b/java/.classpath deleted file mode 100644 index 2ef81d0..0000000 --- a/java/.classpath +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - diff --git a/java/.gitignore b/java/.gitignore index 7c79ae0..752c722 100644 --- a/java/.gitignore +++ b/java/.gitignore @@ -1 +1,3 @@ -TEST-*.xml +/bin +/target +.idea/ diff --git a/java/.project b/java/.project deleted file mode 100644 index 774dcc4..0000000 --- a/java/.project +++ /dev/null @@ -1,23 +0,0 @@ - - - serpent - - - - - - org.eclipse.jdt.core.javabuilder - - - - - org.eclipse.m2e.core.maven2Builder - - - - - - org.eclipse.m2e.core.maven2Nature - org.eclipse.jdt.core.javanature - - diff --git a/java/.settings/org.eclipse.core.resources.prefs b/java/.settings/org.eclipse.core.resources.prefs deleted file mode 100644 index f9fe345..0000000 --- a/java/.settings/org.eclipse.core.resources.prefs +++ /dev/null @@ -1,4 +0,0 @@ -eclipse.preferences.version=1 -encoding//src/main/java=UTF-8 -encoding//src/test/java=UTF-8 -encoding/=UTF-8 diff --git a/java/.settings/org.eclipse.jdt.core.prefs b/java/.settings/org.eclipse.jdt.core.prefs deleted file mode 100644 index 529ef07..0000000 --- a/java/.settings/org.eclipse.jdt.core.prefs +++ /dev/null @@ -1,13 +0,0 @@ -eclipse.preferences.version=1 -org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled -org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 -org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=1.8 -org.eclipse.jdt.core.compiler.debug.lineNumber=generate -org.eclipse.jdt.core.compiler.debug.localVariable=generate -org.eclipse.jdt.core.compiler.debug.sourceFile=generate -org.eclipse.jdt.core.compiler.problem.assertIdentifier=error -org.eclipse.jdt.core.compiler.problem.enumIdentifier=error -org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning -org.eclipse.jdt.core.compiler.source=1.8 diff --git a/java/.settings/org.eclipse.m2e.core.prefs b/java/.settings/org.eclipse.m2e.core.prefs deleted file mode 100644 index f897a7f..0000000 --- a/java/.settings/org.eclipse.m2e.core.prefs +++ /dev/null @@ -1,4 +0,0 @@ -activeProfiles= -eclipse.preferences.version=1 -resolveWorkspaceProjects=true -version=1 diff --git a/java/serpent.iml b/java/serpent.iml new file mode 100644 index 0000000..e39c15d --- /dev/null +++ b/java/serpent.iml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file From 3aa73ef9217801338cd599a32747f992a78d848c Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Fri, 17 Nov 2017 23:00:49 +0100 Subject: [PATCH 119/230] intelliJ tweaks --- .gitignore | 1 - java/.gitignore | 9 ++++++--- java/.idea/.name | 1 + java/.idea/Serpent.iml | 17 +++++++++++++++++ java/.idea/compiler.xml | 17 +++++++++++++++++ java/.idea/encodings.xml | 6 ++++++ .../.idea/libraries/Maven__junit_junit_4_12.xml | 13 +++++++++++++ .../Maven__org_hamcrest_hamcrest_core_1_3.xml | 13 +++++++++++++ java/.idea/misc.xml | 11 +++++++++++ java/.idea/modules.xml | 8 ++++++++ java/.idea/vcs.xml | 6 ++++++ java/src/test/java/SerpentExample.java | 2 +- 12 files changed, 99 insertions(+), 5 deletions(-) create mode 100644 java/.idea/.name create mode 100644 java/.idea/Serpent.iml create mode 100644 java/.idea/compiler.xml create mode 100644 java/.idea/encodings.xml create mode 100644 java/.idea/libraries/Maven__junit_junit_4_12.xml create mode 100644 java/.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_3.xml create mode 100644 java/.idea/misc.xml create mode 100644 java/.idea/modules.xml create mode 100644 java/.idea/vcs.xml diff --git a/.gitignore b/.gitignore index 56df97e..32680ac 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,6 @@ *.so # Packages -*.idea *.egg *.egg-info dist diff --git a/java/.gitignore b/java/.gitignore index 752c722..121b003 100644 --- a/java/.gitignore +++ b/java/.gitignore @@ -1,3 +1,6 @@ -/bin -/target -.idea/ +# intellij workspace info +.idea/workspace.xml + +# java/maven +target/ + diff --git a/java/.idea/.name b/java/.idea/.name new file mode 100644 index 0000000..1c19081 --- /dev/null +++ b/java/.idea/.name @@ -0,0 +1 @@ +Serpent \ No newline at end of file diff --git a/java/.idea/Serpent.iml b/java/.idea/Serpent.iml new file mode 100644 index 0000000..03d554e --- /dev/null +++ b/java/.idea/Serpent.iml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/java/.idea/compiler.xml b/java/.idea/compiler.xml new file mode 100644 index 0000000..a52e7b6 --- /dev/null +++ b/java/.idea/compiler.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/java/.idea/encodings.xml b/java/.idea/encodings.xml new file mode 100644 index 0000000..b26911b --- /dev/null +++ b/java/.idea/encodings.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/java/.idea/libraries/Maven__junit_junit_4_12.xml b/java/.idea/libraries/Maven__junit_junit_4_12.xml new file mode 100644 index 0000000..d411041 --- /dev/null +++ b/java/.idea/libraries/Maven__junit_junit_4_12.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/java/.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_3.xml b/java/.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_3.xml new file mode 100644 index 0000000..f58bbc1 --- /dev/null +++ b/java/.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/java/.idea/misc.xml b/java/.idea/misc.xml new file mode 100644 index 0000000..602a0dd --- /dev/null +++ b/java/.idea/misc.xml @@ -0,0 +1,11 @@ + + + + + + + \ No newline at end of file diff --git a/java/.idea/modules.xml b/java/.idea/modules.xml new file mode 100644 index 0000000..f136caa --- /dev/null +++ b/java/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/java/.idea/vcs.xml b/java/.idea/vcs.xml new file mode 100644 index 0000000..6c0b863 --- /dev/null +++ b/java/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/java/src/test/java/SerpentExample.java b/java/src/test/java/SerpentExample.java index b5af90f..39466fa 100644 --- a/java/src/test/java/SerpentExample.java +++ b/java/src/test/java/SerpentExample.java @@ -96,7 +96,7 @@ public void run() throws IOException System.out.println(""); // parse and print the example file - File testdatafile = new File("test/testserpent.utf8.bin"); + File testdatafile = new File("src/test/java/testserpent.utf8.bin"); ser = new byte[(int) testdatafile.length()]; FileInputStream fis=new FileInputStream(testdatafile); DataInputStream dis = new DataInputStream(fis); From 583e96f0335df29e954d28553bcd4b66fb8bec6e Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Fri, 29 Dec 2017 00:56:46 +0100 Subject: [PATCH 120/230] update year --- LICENSE | 2 +- dotnet/Serpent/Serpent.nuspec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index 0c50aed..ad923a6 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2017 Irmen de Jong +Copyright (c) by Irmen de Jong Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/dotnet/Serpent/Serpent.nuspec b/dotnet/Serpent/Serpent.nuspec index c2944c4..076c7af 100644 --- a/dotnet/Serpent/Serpent.nuspec +++ b/dotnet/Serpent/Serpent.nuspec @@ -23,7 +23,7 @@ CRITICAL FIX: rewrote serialization and parsing of strings containing chars abov Added Serializer.MaximumLevel to avoid too deep recursion resulting in stack overflow errors. Serializer.MaximumLevel decreased to 500. - Copyright 2017 + Copyright 2017, 2018 serialization python pyro From e29e482f6528cc340d12bef526f1eb880ba18573 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Wed, 24 Jan 2018 21:47:41 +0100 Subject: [PATCH 121/230] moved to dotnetcore --- dotnet/Serpent/Version.cs | 15 --------------- .../Serpent/.idea/.idea.Serpent/riderModule.iml | 7 +++++++ .../Serpent/Razorvine.Serpent}/Ast.cs | 10 +--------- .../Serpent/Razorvine.Serpent}/ComplexNumber.cs | 8 -------- .../Serpent/Razorvine.Serpent}/DebugVisitor.cs | 12 +----------- .../Razorvine.Serpent}/ObjectifyVisitor.cs | 10 +--------- .../Serpent/Razorvine.Serpent}/ParseException.cs | 10 +--------- .../Serpent/Razorvine.Serpent}/Parser.cs | 14 +------------- .../Razorvine.Serpent/Razorvine.Serpent.csproj | 6 ++++++ .../Razorvine.Serpent}/SeekableStringReader.cs | 11 +---------- .../Serpent/Razorvine.Serpent}/Serializer.cs | 10 +--------- dotnet_core/Serpent/Razorvine.Serpent/Version.cs | 14 ++++++++++++++ dotnet_core/Serpent/Serpent.sln | 16 ++++++++++++++++ 13 files changed, 50 insertions(+), 93 deletions(-) delete mode 100644 dotnet/Serpent/Version.cs create mode 100644 dotnet_core/Serpent/.idea/.idea.Serpent/riderModule.iml rename {dotnet/Serpent => dotnet_core/Serpent/Razorvine.Serpent}/Ast.cs (96%) rename {dotnet/Serpent => dotnet_core/Serpent/Razorvine.Serpent}/ComplexNumber.cs (88%) rename {dotnet/Serpent => dotnet_core/Serpent/Razorvine.Serpent}/DebugVisitor.cs (87%) rename {dotnet/Serpent => dotnet_core/Serpent/Razorvine.Serpent}/ObjectifyVisitor.cs (91%) rename {dotnet/Serpent => dotnet_core/Serpent/Razorvine.Serpent}/ParseException.cs (54%) rename {dotnet/Serpent => dotnet_core/Serpent/Razorvine.Serpent}/Parser.cs (97%) create mode 100644 dotnet_core/Serpent/Razorvine.Serpent/Razorvine.Serpent.csproj rename {dotnet/Serpent => dotnet_core/Serpent/Razorvine.Serpent}/SeekableStringReader.cs (93%) rename {dotnet/Serpent => dotnet_core/Serpent/Razorvine.Serpent}/Serializer.cs (98%) create mode 100644 dotnet_core/Serpent/Razorvine.Serpent/Version.cs create mode 100644 dotnet_core/Serpent/Serpent.sln diff --git a/dotnet/Serpent/Version.cs b/dotnet/Serpent/Version.cs deleted file mode 100644 index 0d02f6a..0000000 --- a/dotnet/Serpent/Version.cs +++ /dev/null @@ -1,15 +0,0 @@ -/// -/// Serpent, a Python literal expression serializer/deserializer -/// (a.k.a. Python's ast.literal_eval in .NET) -/// -/// Copyright: Irmen de Jong (irmen@razorvine.net) -/// Software license: "MIT software license". See http://opensource.org/licenses/MIT -/// - -namespace Razorvine.Serpent -{ - public static class LibraryVersion - { - public static string Version = "1.18"; - } -} diff --git a/dotnet_core/Serpent/.idea/.idea.Serpent/riderModule.iml b/dotnet_core/Serpent/.idea/.idea.Serpent/riderModule.iml new file mode 100644 index 0000000..1a4e0d9 --- /dev/null +++ b/dotnet_core/Serpent/.idea/.idea.Serpent/riderModule.iml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/dotnet/Serpent/Ast.cs b/dotnet_core/Serpent/Razorvine.Serpent/Ast.cs similarity index 96% rename from dotnet/Serpent/Ast.cs rename to dotnet_core/Serpent/Razorvine.Serpent/Ast.cs index 932c444..c4b5ee1 100644 --- a/dotnet/Serpent/Ast.cs +++ b/dotnet_core/Serpent/Razorvine.Serpent/Ast.cs @@ -1,12 +1,4 @@ -/// -/// Serpent, a Python literal expression serializer/deserializer -/// (a.k.a. Python's ast.literal_eval in .NET) -/// -/// Copyright Irmen de Jong (irmen@razorvine.net) -/// Software license: "MIT software license". See http://opensource.org/licenses/MIT -/// - -using System; +using System; using System.Collections; using System.Collections.Generic; using System.Globalization; diff --git a/dotnet/Serpent/ComplexNumber.cs b/dotnet_core/Serpent/Razorvine.Serpent/ComplexNumber.cs similarity index 88% rename from dotnet/Serpent/ComplexNumber.cs rename to dotnet_core/Serpent/Razorvine.Serpent/ComplexNumber.cs index fc684cc..9858f94 100644 --- a/dotnet/Serpent/ComplexNumber.cs +++ b/dotnet_core/Serpent/Razorvine.Serpent/ComplexNumber.cs @@ -1,11 +1,3 @@ -/// -/// Serpent, a Python literal expression serializer/deserializer -/// (a.k.a. Python's ast.literal_eval in .NET) -/// -/// Copyright Irmen de Jong (irmen@razorvine.net) -/// Software license: "MIT software license". See http://opensource.org/licenses/MIT -/// - using System; using System.Text; diff --git a/dotnet/Serpent/DebugVisitor.cs b/dotnet_core/Serpent/Razorvine.Serpent/DebugVisitor.cs similarity index 87% rename from dotnet/Serpent/DebugVisitor.cs rename to dotnet_core/Serpent/Razorvine.Serpent/DebugVisitor.cs index 1d759b3..37b9a44 100644 --- a/dotnet/Serpent/DebugVisitor.cs +++ b/dotnet_core/Serpent/Razorvine.Serpent/DebugVisitor.cs @@ -1,14 +1,4 @@ -/// -/// Serpent, a Python literal expression serializer/deserializer -/// (a.k.a. Python's ast.literal_eval in .NET) -/// -/// Copyright Irmen de Jong (irmen@razorvine.net) -/// Software license: "MIT software license". See http://opensource.org/licenses/MIT -/// - -using System; -using System.Collections.Generic; -using System.Text; +using System.Text; namespace Razorvine.Serpent { diff --git a/dotnet/Serpent/ObjectifyVisitor.cs b/dotnet_core/Serpent/Razorvine.Serpent/ObjectifyVisitor.cs similarity index 91% rename from dotnet/Serpent/ObjectifyVisitor.cs rename to dotnet_core/Serpent/Razorvine.Serpent/ObjectifyVisitor.cs index 4ba127b..013438b 100644 --- a/dotnet/Serpent/ObjectifyVisitor.cs +++ b/dotnet_core/Serpent/Razorvine.Serpent/ObjectifyVisitor.cs @@ -1,12 +1,4 @@ -/// -/// Serpent, a Python literal expression serializer/deserializer -/// (a.k.a. Python's ast.literal_eval in .NET) -/// -/// Copyright Irmen de Jong (irmen@razorvine.net) -/// Software license: "MIT software license". See http://opensource.org/licenses/MIT -/// - -using System; +using System; using System.Collections.Generic; using System.Collections; diff --git a/dotnet/Serpent/ParseException.cs b/dotnet_core/Serpent/Razorvine.Serpent/ParseException.cs similarity index 54% rename from dotnet/Serpent/ParseException.cs rename to dotnet_core/Serpent/Razorvine.Serpent/ParseException.cs index 3ab1571..c71f074 100644 --- a/dotnet/Serpent/ParseException.cs +++ b/dotnet_core/Serpent/Razorvine.Serpent/ParseException.cs @@ -1,12 +1,4 @@ -/// -/// Serpent, a Python literal expression serializer/deserializer -/// (a.k.a. Python's ast.literal_eval in .NET) -/// -/// Copyright Irmen de Jong (irmen@razorvine.net) -/// Software license: "MIT software license". See http://opensource.org/licenses/MIT -/// - -using System; +using System; namespace Razorvine.Serpent { diff --git a/dotnet/Serpent/Parser.cs b/dotnet_core/Serpent/Razorvine.Serpent/Parser.cs similarity index 97% rename from dotnet/Serpent/Parser.cs rename to dotnet_core/Serpent/Razorvine.Serpent/Parser.cs index 6c4851b..c19b7ac 100644 --- a/dotnet/Serpent/Parser.cs +++ b/dotnet_core/Serpent/Razorvine.Serpent/Parser.cs @@ -1,15 +1,5 @@ -/// -/// Serpent, a Python literal expression serializer/deserializer -/// (a.k.a. Python's ast.literal_eval in .NET) -/// -/// Copyright Irmen de Jong (irmen@razorvine.net) -/// Software license: "MIT software license". See http://opensource.org/licenses/MIT -/// - -using System; -#if !(SILVERLIGHT || WINDOWS_PHONE || PORTABLE) +using System; using System.Collections; -#endif using System.Collections.Generic; using System.Globalization; using System.Text; @@ -630,7 +620,6 @@ double ParseDouble(string numberstr) /// If it is something else, throw an ArgumentException /// public static byte[] ToBytes(object obj) { -#if !(SILVERLIGHT || WINDOWS_PHONE || PORTABLE) Hashtable hashtable = obj as Hashtable; if(hashtable!=null) { @@ -644,7 +633,6 @@ public static byte[] ToBytes(object obj) { } return Convert.FromBase64String(data); } -#endif IDictionary dict = obj as IDictionary; if(dict!=null) diff --git a/dotnet_core/Serpent/Razorvine.Serpent/Razorvine.Serpent.csproj b/dotnet_core/Serpent/Razorvine.Serpent/Razorvine.Serpent.csproj new file mode 100644 index 0000000..31b1cfc --- /dev/null +++ b/dotnet_core/Serpent/Razorvine.Serpent/Razorvine.Serpent.csproj @@ -0,0 +1,6 @@ + + + netstandard2.0 + 6 + + \ No newline at end of file diff --git a/dotnet/Serpent/SeekableStringReader.cs b/dotnet_core/Serpent/Razorvine.Serpent/SeekableStringReader.cs similarity index 93% rename from dotnet/Serpent/SeekableStringReader.cs rename to dotnet_core/Serpent/Razorvine.Serpent/SeekableStringReader.cs index 5e91128..bd85542 100644 --- a/dotnet/Serpent/SeekableStringReader.cs +++ b/dotnet_core/Serpent/Razorvine.Serpent/SeekableStringReader.cs @@ -1,13 +1,4 @@ -/// -/// Serpent, a Python literal expression serializer/deserializer -/// (a.k.a. Python's ast.literal_eval in .NET) -/// -/// Copyright Irmen de Jong (irmen@razorvine.net) -/// Software license: "MIT software license". See http://opensource.org/licenses/MIT -/// - -using System; -using System.Collections.Generic; +using System; namespace Razorvine.Serpent { diff --git a/dotnet/Serpent/Serializer.cs b/dotnet_core/Serpent/Razorvine.Serpent/Serializer.cs similarity index 98% rename from dotnet/Serpent/Serializer.cs rename to dotnet_core/Serpent/Razorvine.Serpent/Serializer.cs index 7b9f61f..fd01ee9 100644 --- a/dotnet/Serpent/Serializer.cs +++ b/dotnet_core/Serpent/Razorvine.Serpent/Serializer.cs @@ -1,12 +1,4 @@ -/// -/// Serpent, a Python literal expression serializer/deserializer -/// (a.k.a. Python's ast.literal_eval in .NET) -/// -/// Copyright Irmen de Jong (irmen@razorvine.net) -/// Software license: "MIT software license". See http://opensource.org/licenses/MIT -/// - -using System; +using System; using System.Collections; using System.Collections.Generic; using System.Globalization; diff --git a/dotnet_core/Serpent/Razorvine.Serpent/Version.cs b/dotnet_core/Serpent/Razorvine.Serpent/Version.cs new file mode 100644 index 0000000..6730e39 --- /dev/null +++ b/dotnet_core/Serpent/Razorvine.Serpent/Version.cs @@ -0,0 +1,14 @@ +/*** +Serpent, a Python literal expression serializer/deserializer +(a.k.a. Python's ast.literal_eval in .NET) +Copyright: Irmen de Jong (irmen@razorvine.net) +Software license: "MIT software license". See http://opensource.org/licenses/MIT +***/ + +namespace Razorvine.Serpent +{ + public static class LibraryVersion + { + public static string Version = "1.20"; + } +} diff --git a/dotnet_core/Serpent/Serpent.sln b/dotnet_core/Serpent/Serpent.sln new file mode 100644 index 0000000..edb186e --- /dev/null +++ b/dotnet_core/Serpent/Serpent.sln @@ -0,0 +1,16 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Razorvine.Serpent", "Razorvine.Serpent\Razorvine.Serpent.csproj", "{2FD56EEF-23E7-4CE8-94FA-92B0FC6E5CA4}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {2FD56EEF-23E7-4CE8-94FA-92B0FC6E5CA4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2FD56EEF-23E7-4CE8-94FA-92B0FC6E5CA4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2FD56EEF-23E7-4CE8-94FA-92B0FC6E5CA4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2FD56EEF-23E7-4CE8-94FA-92B0FC6E5CA4}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal From 0d01e196ec3e851b8b9712eec896f069214c4282 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Wed, 24 Jan 2018 22:00:13 +0100 Subject: [PATCH 122/230] moved to dotnetcore and switched from nunit to mstest --- .../.idea/.idea.Serpent/riderModule.iml | 3 + dotnet_core/Serpent/Serpent.sln | 6 + .../Serpent/Tests}/CycleTest.cs | 18 +-- .../Serpent/Tests}/Example.cs | 8 +- .../Serpent/Tests}/ParserTest.cs | 108 +++++++++--------- .../Serpent/Tests}/SerializeTest.cs | 72 ++++++------ .../Serpent/Tests}/SlowPerformanceTest.cs | 8 +- .../Serpent/Tests}/StringreaderTest.cs | 34 +++--- dotnet_core/Serpent/Tests/Tests.csproj | 20 ++++ .../Serpent/Tests}/testserpent.utf8.bin | 0 10 files changed, 153 insertions(+), 124 deletions(-) rename {dotnet/Serpent.Test => dotnet_core/Serpent/Tests}/CycleTest.cs (93%) rename {dotnet/Serpent.Test => dotnet_core/Serpent/Tests}/Example.cs (96%) rename {dotnet/Serpent.Test => dotnet_core/Serpent/Tests}/ParserTest.cs (92%) rename {dotnet/Serpent.Test => dotnet_core/Serpent/Tests}/SerializeTest.cs (95%) rename {dotnet/Serpent.Test => dotnet_core/Serpent/Tests}/SlowPerformanceTest.cs (94%) rename {dotnet/Serpent.Test => dotnet_core/Serpent/Tests}/StringreaderTest.cs (85%) create mode 100644 dotnet_core/Serpent/Tests/Tests.csproj rename {dotnet/Serpent.Test => dotnet_core/Serpent/Tests}/testserpent.utf8.bin (100%) diff --git a/dotnet_core/Serpent/.idea/.idea.Serpent/riderModule.iml b/dotnet_core/Serpent/.idea/.idea.Serpent/riderModule.iml index 1a4e0d9..84dbb3d 100644 --- a/dotnet_core/Serpent/.idea/.idea.Serpent/riderModule.iml +++ b/dotnet_core/Serpent/.idea/.idea.Serpent/riderModule.iml @@ -1,6 +1,9 @@ + + + diff --git a/dotnet_core/Serpent/Serpent.sln b/dotnet_core/Serpent/Serpent.sln index edb186e..af4321a 100644 --- a/dotnet_core/Serpent/Serpent.sln +++ b/dotnet_core/Serpent/Serpent.sln @@ -2,6 +2,8 @@ Microsoft Visual Studio Solution File, Format Version 12.00 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Razorvine.Serpent", "Razorvine.Serpent\Razorvine.Serpent.csproj", "{2FD56EEF-23E7-4CE8-94FA-92B0FC6E5CA4}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tests", "Tests\Tests.csproj", "{CF817063-FC62-4E9D-B6FC-9E90390BEC32}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -12,5 +14,9 @@ Global {2FD56EEF-23E7-4CE8-94FA-92B0FC6E5CA4}.Debug|Any CPU.Build.0 = Debug|Any CPU {2FD56EEF-23E7-4CE8-94FA-92B0FC6E5CA4}.Release|Any CPU.ActiveCfg = Release|Any CPU {2FD56EEF-23E7-4CE8-94FA-92B0FC6E5CA4}.Release|Any CPU.Build.0 = Release|Any CPU + {CF817063-FC62-4E9D-B6FC-9E90390BEC32}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CF817063-FC62-4E9D-B6FC-9E90390BEC32}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CF817063-FC62-4E9D-B6FC-9E90390BEC32}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CF817063-FC62-4E9D-B6FC-9E90390BEC32}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/dotnet/Serpent.Test/CycleTest.cs b/dotnet_core/Serpent/Tests/CycleTest.cs similarity index 93% rename from dotnet/Serpent.Test/CycleTest.cs rename to dotnet_core/Serpent/Tests/CycleTest.cs index f16ab58..ebb9950 100644 --- a/dotnet/Serpent.Test/CycleTest.cs +++ b/dotnet_core/Serpent/Tests/CycleTest.cs @@ -9,14 +9,14 @@ using System; using System.Collections; using System.Collections.Generic; -using NUnit.Framework; +using Microsoft.VisualStudio.TestTools.UnitTesting; namespace Razorvine.Serpent.Test { - [TestFixture] + [TestClass] public class CycleTest { - [Test] + [TestMethod] public void testTupleOk() { var ser = new Serializer(); @@ -27,7 +27,7 @@ public void testTupleOk() parser.Parse(data); } - [Test] + [TestMethod] public void testListOk() { var ser = new Serializer(); @@ -44,7 +44,7 @@ public void testListOk() parser.Parse(data); } - [Test] + [TestMethod] public void testDictOk() { var ser = new Serializer(); @@ -59,7 +59,7 @@ public void testDictOk() parser.Parse(data); } - [Test] + [TestMethod] [ExpectedException(typeof(ArgumentException))] public void testListCycle() { @@ -71,7 +71,7 @@ public void testListCycle() ser.Serialize(d); } - [Test] + [TestMethod] [ExpectedException(typeof(ArgumentException))] public void testDictCycle() { @@ -83,7 +83,7 @@ public void testDictCycle() ser.Serialize(d); } - [Test] + [TestMethod] [ExpectedException(typeof(ArgumentException))] public void testClassCycle() { @@ -96,7 +96,7 @@ public void testClassCycle() ser.Serialize(d); } - [Test] + [TestMethod] public void testMaxLevel() { Serializer ser = new Serializer(); diff --git a/dotnet/Serpent.Test/Example.cs b/dotnet_core/Serpent/Tests/Example.cs similarity index 96% rename from dotnet/Serpent.Test/Example.cs rename to dotnet_core/Serpent/Tests/Example.cs index 6d18372..b0e3110 100644 --- a/dotnet/Serpent.Test/Example.cs +++ b/dotnet_core/Serpent/Tests/Example.cs @@ -4,7 +4,7 @@ using System.Text; using System.Linq; using System.IO; -using NUnit.Framework; +using Microsoft.VisualStudio.TestTools.UnitTesting; using Razorvine.Serpent; namespace Razorvine.Serpent.Test @@ -12,11 +12,11 @@ namespace Razorvine.Serpent.Test /// /// Example usage. /// - [TestFixture] - [Explicit("example")] + [TestClass] + [Ignore("example")] public class Example { - [Test] + [TestMethod] public void ExampleUsage() { Console.WriteLine("using serpent library version {0}", LibraryVersion.Version); diff --git a/dotnet/Serpent.Test/ParserTest.cs b/dotnet_core/Serpent/Tests/ParserTest.cs similarity index 92% rename from dotnet/Serpent.Test/ParserTest.cs rename to dotnet_core/Serpent/Tests/ParserTest.cs index bc59b1b..7655b29 100644 --- a/dotnet/Serpent.Test/ParserTest.cs +++ b/dotnet_core/Serpent/Tests/ParserTest.cs @@ -11,14 +11,14 @@ using System.Collections.Generic; using System.IO; using System.Text; -using NUnit.Framework; +using Microsoft.VisualStudio.TestTools.UnitTesting; namespace Razorvine.Serpent.Test { - [TestFixture] + [TestClass] public class ParserTest { - [Test] + [TestMethod] public void TestBasic() { Parser p = new Parser(); @@ -27,7 +27,7 @@ public void TestBasic() Assert.IsNotNull(p.Parse("# comment\n42\n").Root); } - [Test] + [TestMethod] public void TestComments() { Parser p = new Parser(); @@ -50,7 +50,7 @@ public void TestComments() Assert.AreEqual(new int[] {1,2,3,4}, obj); } - [Test] + [TestMethod] public void TestPrimitives() { Parser p = new Parser(); @@ -70,10 +70,10 @@ public void TestPrimitives() Assert.AreEqual(new Ast.DecimalNode(123456789123456789123456789M), p.Parse("123456789123456789123456789").Root); Assert.AreNotEqual(new Ast.LongNode(52), p.Parse("52").Root); Assert.AreEqual(new Ast.LongNode(123456789123456789L), p.Parse("123456789123456789").Root); - Assert.Throws(()=>p.Parse("123456789123456789123456789123456789")); // overflow + Assert.ThrowsException(()=>p.Parse("123456789123456789123456789123456789")); // overflow } - [Test] + [TestMethod] public void TestWeirdFloats() { Parser p = new Parser(); @@ -96,7 +96,7 @@ public void TestWeirdFloats() Assert.IsTrue(double.IsNegativeInfinity(c.Imaginary)); } - [Test] + [TestMethod] public void TestFloatPrecision() { Parser p = new Parser(); @@ -122,7 +122,7 @@ public void TestFloatPrecision() Assert.AreEqual((-98765432123456.12345678987656e-44).ToString(), dv.Value.ToString()); } - [Test] + [TestMethod] public void TestEquality() { Ast.INode n1, n2; @@ -258,7 +258,7 @@ public void TestEquality() } - [Test] + [TestMethod] public void TestDictEquality() { Ast.DictNode dict1 = new Ast.DictNode(); @@ -304,7 +304,7 @@ public void TestDictEquality() Assert.AreNotEqual(dict1, dict2); } - [Test] + [TestMethod] public void TestSetEquality() { Ast.SetNode set1 = new Ast.SetNode(); @@ -322,7 +322,7 @@ public void TestSetEquality() Assert.AreNotEqual(set1, set2); } - [Test] + [TestMethod] public void TestPrintSingle() { Parser p = new Parser(); @@ -354,7 +354,7 @@ public void TestPrintSingle() Assert.AreEqual("123456789123456789123456789", p.Parse("123456789123456789123456789").Root.ToString()); } - [Test] + [TestMethod] public void TestPrintSeq() { Parser p=new Parser(); @@ -380,23 +380,23 @@ public void TestPrintSeq() Assert.AreEqual("{'a':42,'b':45}", p.Parse("{'a': 42, 'b': 43, 'b': 44, 'b': 45}").Root.ToString()); } - [Test] + [TestMethod] public void TestInvalidPrimitives() { Parser p = new Parser(); - Assert.Throws(()=>p.Parse("1+2")); - Assert.Throws(()=>p.Parse("1-2")); - Assert.Throws(()=>p.Parse("1.1+2.2")); - Assert.Throws(()=>p.Parse("1.1-2.2")); - Assert.Throws(()=>p.Parse("True+2")); - Assert.Throws(()=>p.Parse("False-2")); - Assert.Throws(()=>p.Parse("3j+2")); - Assert.Throws(()=>p.Parse("3j-2")); - Assert.Throws(()=>p.Parse("None+2")); - Assert.Throws(()=>p.Parse("None-2")); + Assert.ThrowsException(()=>p.Parse("1+2")); + Assert.ThrowsException(()=>p.Parse("1-2")); + Assert.ThrowsException(()=>p.Parse("1.1+2.2")); + Assert.ThrowsException(()=>p.Parse("1.1-2.2")); + Assert.ThrowsException(()=>p.Parse("True+2")); + Assert.ThrowsException(()=>p.Parse("False-2")); + Assert.ThrowsException(()=>p.Parse("3j+2")); + Assert.ThrowsException(()=>p.Parse("3j-2")); + Assert.ThrowsException(()=>p.Parse("None+2")); + Assert.ThrowsException(()=>p.Parse("None-2")); } - [Test] + [TestMethod] public void TestComplex() { Parser p = new Parser(); @@ -444,7 +444,7 @@ public void TestComplex() Assert.AreEqual(cplx, p.Parse("(-3.2e-32-9.9e-44j)").Root); } - [Test] + [TestMethod] public void TestComplexPrecision() { Parser p = new Parser(); @@ -462,7 +462,7 @@ public void TestComplexPrecision() Assert.AreEqual(-665544332211.9998877665544e+44, cv.Imaginary); } - [Test] + [TestMethod] public void TestPrimitivesStuffAtEnd() { Parser p = new Parser(); @@ -479,7 +479,7 @@ public void TestPrimitivesStuffAtEnd() Assert.AreEqual(cplx, p.ParseSingle(new SeekableStringReader("3j@"))); } - [Test] + [TestMethod] public void TestStrings() { Parser p = new Parser(); @@ -492,7 +492,7 @@ public void TestStrings() Assert.AreEqual(new Ast.StringNode("tab\tnewline\n."), p.Parse("'tab\\tnewline\\n.'").Root); } - [Test] + [TestMethod] public void TestUnicode() { Parser p = new Parser(); @@ -506,7 +506,7 @@ public void TestUnicode() Assert.AreEqual(new Ast.StringNode(value), p.Parse(bytes).Root); } - [Test] + [TestMethod] public void TestLongUnicodeRoundtrip() { Char[] chars64k = new Char[65536]; @@ -517,14 +517,14 @@ public void TestLongUnicodeRoundtrip() Serializer ser=new Serializer(); byte[] data = ser.Serialize(str64k); - Assert.Greater(data.Length, chars64k.Length); + Assert.IsTrue(data.Length > chars64k.Length); Parser p=new Parser(); String result = (String)p.Parse(data).GetData(); Assert.AreEqual(str64k, result); } - [Test] + [TestMethod] public void TestWhitespace() { Parser p = new Parser(); @@ -533,7 +533,7 @@ public void TestWhitespace() Assert.AreEqual(new Ast.IntegerNode(42), p.Parse("\t42\r\n").Root); Assert.AreEqual(new Ast.IntegerNode(42), p.Parse(" \t 42 \r \n ").Root); Assert.AreEqual(new Ast.StringNode(" string value "), p.Parse(" ' string value ' ").Root); - Assert.Throws(()=>p.Parse(" ( 42 , ( 'x', 'y' ) ")); // missing tuple close ) + Assert.ThrowsException(()=>p.Parse(" ( 42 , ( 'x', 'y' ) ")); // missing tuple close ) Ast ast = p.Parse(" ( 42 , ( 'x', 'y' ) ) "); Ast.TupleNode tuple = (Ast.TupleNode) ast.Root; Assert.AreEqual(new Ast.IntegerNode(42), tuple.Elements[0]); @@ -547,7 +547,7 @@ public void TestWhitespace() p.Parse(" { 52 } "); } - [Test] + [TestMethod] public void TestTuple() { Parser p = new Parser(); @@ -569,12 +569,12 @@ public void TestTuple() Assert.AreEqual(tuple, p.Parse("(42,)").Root); Assert.AreEqual(tuple2, p.Parse("( 42,43, 44 )").Root); - Assert.Throws(()=>p.Parse("(42,43]")); - Assert.Throws(()=>p.Parse("()@")); - Assert.Throws(()=>p.Parse("(42,43)@")); + Assert.ThrowsException(()=>p.Parse("(42,43]")); + Assert.ThrowsException(()=>p.Parse("()@")); + Assert.ThrowsException(()=>p.Parse("(42,43)@")); } - [Test] + [TestMethod] public void TestList() { Parser p = new Parser(); @@ -596,12 +596,12 @@ public void TestList() Assert.AreEqual(list, p.Parse("[42]").Root); Assert.AreEqual(list2, p.Parse("[ 42,43, 44 ]").Root); - Assert.Throws(()=>p.Parse("[42,43}")); - Assert.Throws(()=>p.Parse("[]@")); - Assert.Throws(()=>p.Parse("[42,43]@")); + Assert.ThrowsException(()=>p.Parse("[42,43}")); + Assert.ThrowsException(()=>p.Parse("[]@")); + Assert.ThrowsException(()=>p.Parse("[42,43]@")); } - [Test] + [TestMethod] public void TestSet() { Parser p = new Parser(); @@ -623,8 +623,8 @@ public void TestSet() Assert.AreEqual(set1, p.Parse("{42}").Root); Assert.AreEqual(set2, p.Parse("{ 42,43, 44 }").Root); - Assert.Throws(()=>p.Parse("{42,43]")); - Assert.Throws(()=>p.Parse("{42,43}@")); + Assert.ThrowsException(()=>p.Parse("{42,43]")); + Assert.ThrowsException(()=>p.Parse("{42,43}@")); set1 = p.Parse("{'first','second','third','fourth','fifth','second', 'first', 'third', 'third' }").Root as Ast.SetNode; Assert.AreEqual("'first'", set1.Elements[0].ToString()); @@ -635,7 +635,7 @@ public void TestSet() Assert.AreEqual(5, set1.Elements.Count); } - [Test] + [TestMethod] public void TestDict() { Parser p = new Parser(); @@ -664,9 +664,9 @@ public void TestDict() Assert.AreEqual(dict1, p.Parse("{'key1': 42}").Root); Assert.AreEqual(dict2, p.Parse("{'key1': 42, 'key2': 43, 'key3':44}").Root); - Assert.Throws(()=>p.Parse("{'key': 42]")); - Assert.Throws(()=>p.Parse("{}@")); - Assert.Throws(()=>p.Parse("{'key': 42}@")); + Assert.ThrowsException(()=>p.Parse("{'key': 42]")); + Assert.ThrowsException(()=>p.Parse("{}@")); + Assert.ThrowsException(()=>p.Parse("{'key': 42}@")); dict1 = p.Parse("{'a': 1, 'b': 2, 'c': 3, 'c': 4, 'c': 5, 'c': 6}").Root as Ast.DictNode; Assert.AreEqual("'a':1", dict1.Elements[0].ToString()); @@ -675,7 +675,7 @@ public void TestDict() Assert.AreEqual(3, dict1.Elements.Count); } - [Test] + [TestMethod] public void TestFile() { Parser p = new Parser(); @@ -700,7 +700,7 @@ public void TestFile() // @TODO Assert.AreEqual(ast.Root, ast2.Root); } - [Test] + [TestMethod] [Ignore("can't yet get the ast to compare equal on mono")] public void TestAstEquals() { @@ -725,7 +725,7 @@ public void Walk(Ast.INode node, StringBuilder sb) sb.AppendLine(string.Format("{0} = {1}", node.GetType(), node.ToString())); } - [Test] + [TestMethod] public void TestTrailingCommas() { Parser p = new Parser(); @@ -762,10 +762,10 @@ public void TestTrailingCommas() } } - [TestFixture] + [TestClass] public class VisitorTest { - [Test] + [TestMethod] public void TestObjectify() { Parser p = new Parser(); @@ -800,7 +800,7 @@ object ZerodivisionFromDict(IDictionary dict) return null; } - [Test] + [TestMethod] public void TestObjectifyDictToClass() { Parser p = new Parser(); diff --git a/dotnet/Serpent.Test/SerializeTest.cs b/dotnet_core/Serpent/Tests/SerializeTest.cs similarity index 95% rename from dotnet/Serpent.Test/SerializeTest.cs rename to dotnet_core/Serpent/Tests/SerializeTest.cs index bf0346e..adce5e1 100644 --- a/dotnet/Serpent.Test/SerializeTest.cs +++ b/dotnet_core/Serpent/Tests/SerializeTest.cs @@ -10,13 +10,13 @@ using System.Collections.Generic; using System.Text; -using NUnit.Framework; +using Microsoft.VisualStudio.TestTools.UnitTesting; using Hashtable = System.Collections.Hashtable; using IDictionary = System.Collections.IDictionary; namespace Razorvine.Serpent.Test { - [TestFixture] + [TestClass] public class SerializeTest { public byte[] strip_header(byte[] data) @@ -41,7 +41,7 @@ public string S(byte[] b) } - [Test] + [TestMethod] public void TestHeader() { Serializer ser = new Serializer(); @@ -61,7 +61,7 @@ public void TestHeader() } - [Test] + [TestMethod] public void TestStuff() { Serializer ser=new Serializer(); @@ -76,7 +76,7 @@ public void TestStuff() Assert.AreEqual(B("'123456789.98765432198765432199'"), result); } - [Test] + [TestMethod] public void TestNull() { Serializer ser = new Serializer(); @@ -85,7 +85,7 @@ public void TestNull() Assert.AreEqual(B("None"),data); } - [Test] + [TestMethod] public void TestStrings() { Serializer serpent = new Serializer(); @@ -100,7 +100,7 @@ public void TestStrings() Assert.AreEqual(B("\"quotes2'\""), data); } - [Test] + [TestMethod] public void TestUnicodeEscapes() { Serializer serpent=new Serializer(); @@ -166,7 +166,7 @@ public void TestUnicodeEscapes() 39}, data); } - [Test] + [TestMethod] public void TestNumbers() { Serializer serpent = new Serializer(); @@ -205,7 +205,7 @@ public void TestNumbers() Assert.AreEqual(B("(-2-3j)"), data); } - [Test] + [TestMethod] public void TestDoubleNanInf() { Serializer serpent = new Serializer(); @@ -217,7 +217,7 @@ public void TestDoubleNanInf() Assert.AreEqual("(1e30000,-1e30000,{'__class__':'float','value':'nan'},1e30000,-1e30000,{'__class__':'float','value':'nan'},(1e30000+3.4j))", S(data)); } - [Test] + [TestMethod] public void TestBool() { Serializer serpent = new Serializer(); @@ -229,7 +229,7 @@ public void TestBool() Assert.AreEqual(B("False"),data); } - [Test] + [TestMethod] public void TestList() { Serializer serpent = new Serializer(); @@ -254,7 +254,7 @@ public void TestList() Assert.AreEqual("[\n 42,\n 'Sally',\n 16.5\n]", S(ser)); } - [Test] + [TestMethod] public void TestSet() { // test with set literals @@ -287,7 +287,7 @@ public void TestSet() Assert.AreEqual("(42,'Sally',16.5)", S(ser)); // needs to be tuple now } - [Test] + [TestMethod] public void TestDictionary() { Serializer serpent = new Serializer(); @@ -347,7 +347,7 @@ public void TestDictionary() Assert.IsTrue(ser_str=="{2:'two',1:'one'}" || ser_str=="{1:'one',2:'two'}"); } - [Test] + [TestMethod] public void TestBytes() { Serializer serpent = new Serializer(indent: true); @@ -381,20 +381,20 @@ public void TestBytes() Assert.AreEqual(bytes, bytes2); dict["encoding"] = "base99"; - Assert.Throws(()=>Parser.ToBytes(dict)); + Assert.ThrowsException(()=>Parser.ToBytes(dict)); dict.Clear(); - Assert.Throws(()=>Parser.ToBytes(dict)); + Assert.ThrowsException(()=>Parser.ToBytes(dict)); dict.Clear(); dict["data"] = "YWJjZGVm"; - Assert.Throws(()=>Parser.ToBytes(dict)); + Assert.ThrowsException(()=>Parser.ToBytes(dict)); dict.Clear(); dict["encoding"] = "base64"; - Assert.Throws(()=>Parser.ToBytes(dict)); - Assert.Throws(()=>Parser.ToBytes(12345)); - Assert.Throws(()=>Parser.ToBytes(null)); + Assert.ThrowsException(()=>Parser.ToBytes(dict)); + Assert.ThrowsException(()=>Parser.ToBytes(12345)); + Assert.ThrowsException(()=>Parser.ToBytes(null)); } - [Test] + [TestMethod] public void TestCollection() { ICollection intlist = new LinkedList(); @@ -420,7 +420,7 @@ public void TestCollection() } - [Test] + [TestMethod] public void TestIndentation() { var dict = new Dictionary(); @@ -464,7 +464,7 @@ public void TestIndentation() Assert.AreEqual(txt, ser_txt); } - [Test] + [TestMethod] public void TestSorting() { Serializer serpent=new Serializer(); @@ -507,7 +507,7 @@ public void TestSorting() Assert.AreEqual("{\n 'a',\n 'b',\n 'c',\n 'x',\n 'y',\n 'z'\n}", S(ser)); } - [Test] + [TestMethod] public void TestClass() { Serializer.RegisterClass(typeof(SerializeTestClass), null); @@ -522,7 +522,7 @@ public void TestClass() Assert.AreEqual("{\n '__class__': 'SerializeTestClass',\n 'i': 99,\n 'obj': None,\n 's': 'hi'\n}", S(ser)); } - [Test] + [TestMethod] public void TestClass2() { Serializer.RegisterClass(typeof(SerializeTestClass), null); @@ -547,7 +547,7 @@ protected IDictionary testclassConverter(object obj) return result; } - [Test] + [TestMethod] public void TestCustomClassDict() { Serializer.RegisterClass(typeof(SerializeTestClass), testclassConverter); @@ -562,7 +562,7 @@ public void TestCustomClassDict() Assert.AreEqual("{\n '__class@__': 'SerializeTestClass@',\n 'i@': 99,\n 's@': 'hi',\n 'x@': 42\n}", S(ser)); } - [Test] + [TestMethod] public void TestStruct() { Serializer serpent = new Serializer(indent: true); @@ -576,7 +576,7 @@ public void TestStruct() Assert.AreEqual("{\n '__class__': 'SerializeTestStruct',\n 'i': 99,\n 's': 'hi'\n}", S(ser)); } - [Test] + [TestMethod] public void TestStruct2() { Serializer serpent = new Serializer(indent: true, namespaceInClassName: true); @@ -590,7 +590,7 @@ public void TestStruct2() Assert.AreEqual("{\n '__class__': 'Razorvine.Serpent.Test.SerializeTestStruct',\n 'i': 99,\n 's': 'hi'\n}", S(ser)); } - [Test] + [TestMethod] public void TestAnonymousClass() { Serializer serpent = new Serializer(indent: true); @@ -604,7 +604,7 @@ public void TestAnonymousClass() Assert.AreEqual("{\n 'Age': 33,\n 'Country': 'NL',\n 'Name': 'Harry'\n}", S(ser)); } - [Test] + [TestMethod] public void TestDateTime() { Serializer serpent = new Serializer(); @@ -630,7 +630,7 @@ public void TestDateTime() Assert.AreEqual("123630.999", S(ser)); } - [Test] + [TestMethod] public void TestDateTimeOffset() { Serializer serpent = new Serializer(); @@ -644,7 +644,7 @@ public void TestDateTimeOffset() Assert.AreEqual("'2013-05-10T13:59:45+02:00'", S(ser)); } - [Test] + [TestMethod] public void TestException() { Exception x = new ApplicationException("errormessage"); @@ -657,7 +657,7 @@ public void TestException() Assert.AreEqual("{\n '__class__': 'ApplicationException',\n '__exception__': True,\n 'args': (\n 'errormessage',\n ),\n 'attributes': {\n 'custom_attribute': 999\n }\n}", S(ser)); } - [Test] + [TestMethod] public void TestExceptionWithNamespace() { Exception x = new ApplicationException("errormessage"); @@ -671,7 +671,7 @@ enum FooType { Jarjar } - [Test] + [TestMethod] public void TestEnum() { FooType e = FooType.Jarjar; @@ -697,7 +697,7 @@ protected IDictionary AnyClassSerializer(object arg) return result; } - [Test] + [TestMethod] public void testAbstractBaseClassHierarchyPickler() { ConcreteSubClass c = new ConcreteSubClass(); @@ -709,7 +709,7 @@ public void testAbstractBaseClassHierarchyPickler() Assert.AreEqual("{'(SUB)CLASS':'ConcreteSubClass'}", S(strip_header(data))); } - [Test] + [TestMethod] public void TestInterfaceHierarchyPickler() { BaseClassWithInterface b = new BaseClassWithInterface(); diff --git a/dotnet/Serpent.Test/SlowPerformanceTest.cs b/dotnet_core/Serpent/Tests/SlowPerformanceTest.cs similarity index 94% rename from dotnet/Serpent.Test/SlowPerformanceTest.cs rename to dotnet_core/Serpent/Tests/SlowPerformanceTest.cs index 7550ec3..30dc78c 100644 --- a/dotnet/Serpent.Test/SlowPerformanceTest.cs +++ b/dotnet_core/Serpent/Tests/SlowPerformanceTest.cs @@ -1,15 +1,15 @@ using System; -using NUnit.Framework; +using Microsoft.VisualStudio.TestTools.UnitTesting; namespace Razorvine.Serpent.Test { -[TestFixture] +[TestClass] public class SlowPerformanceTest { // tests some performance regressions when they occur - [Test] + [TestMethod] [Ignore("number parse performance in long lists has been resolved")] public void TestManyFloats() { @@ -30,7 +30,7 @@ public void TestManyFloats() Console.WriteLine(""+duration+" valuelen="+values.Length); } - [Test] + [TestMethod] [Ignore("number parse performance in long lists has been resolved")] public void TestManyInts() { diff --git a/dotnet/Serpent.Test/StringreaderTest.cs b/dotnet_core/Serpent/Tests/StringreaderTest.cs similarity index 85% rename from dotnet/Serpent.Test/StringreaderTest.cs rename to dotnet_core/Serpent/Tests/StringreaderTest.cs index 7a5f94c..2c59187 100644 --- a/dotnet/Serpent.Test/StringreaderTest.cs +++ b/dotnet_core/Serpent/Tests/StringreaderTest.cs @@ -8,14 +8,14 @@ using System; using System.Collections.Generic; -using NUnit.Framework; +using Microsoft.VisualStudio.TestTools.UnitTesting; namespace Razorvine.Serpent.Test { - [TestFixture] + [TestClass] public class StringreaderTest { - [Test] + [TestMethod] public void TestStuff() { using(SeekableStringReader s=new SeekableStringReader("hello")) @@ -36,11 +36,11 @@ public void TestStuff() Assert.AreEqual("whitespace", s2.ReadUntil('.')); s2.SkipWhitespace(); Assert.IsFalse(s2.HasMore()); - Assert.Throws(()=>s2.Peek()); + Assert.ThrowsException(()=>s2.Peek()); } } - [Test] + [TestMethod] public void TestRead() { SeekableStringReader s = new SeekableStringReader("hello"); @@ -48,40 +48,40 @@ public void TestRead() Assert.AreEqual('e', s.Read()); Assert.AreEqual("l", s.Read(1)); Assert.AreEqual("lo", s.Read(2)); - Assert.Throws(()=>s.Read()); + Assert.ThrowsException(()=>s.Read()); } - [Test] + [TestMethod] public void TestRanges() { SeekableStringReader s = new SeekableStringReader("hello"); - Assert.Throws(()=>s.Read(-1)); + Assert.ThrowsException(()=>s.Read(-1)); Assert.AreEqual("hello", s.Read(999)); - Assert.Throws(()=>s.Read(1)); + Assert.ThrowsException(()=>s.Read(1)); s.Rewind(int.MaxValue); Assert.IsTrue(s.HasMore()); Assert.AreEqual("hello", s.Peek(999)); Assert.IsTrue(s.HasMore()); } - [Test] + [TestMethod] public void TestReadUntil() { SeekableStringReader s = new SeekableStringReader("hello there"); s.Read(); Assert.AreEqual("ello", s.ReadUntil(' ')); Assert.AreEqual('t', s.Peek()); - Assert.Throws(()=>s.ReadUntil('x')); + Assert.ThrowsException(()=>s.ReadUntil('x')); Assert.AreEqual("there", s.Rest()); - Assert.Throws(()=>s.Rest()); + Assert.ThrowsException(()=>s.Rest()); s.Rewind(int.MaxValue); Assert.AreEqual("hell", s.ReadUntil('x', 'y', 'z', ' ', 'o')); - Assert.Throws(()=>s.ReadUntil('x', 'y', '@')); + Assert.ThrowsException(()=>s.ReadUntil('x', 'y', '@')); } - [Test] + [TestMethod] public void TestReadWhile() { SeekableStringReader s = new SeekableStringReader("123.456 foo"); @@ -91,7 +91,7 @@ public void TestReadWhile() Assert.AreEqual("foo", s.Rest()); } - [Test] + [TestMethod] public void TestBookmark() { SeekableStringReader s = new SeekableStringReader("hello"); @@ -103,7 +103,7 @@ public void TestBookmark() Assert.AreEqual("o", s.Read(999)); } - [Test] + [TestMethod] public void TestNesting() { SeekableStringReader outer = new SeekableStringReader("hello!"); @@ -122,7 +122,7 @@ public void TestNesting() Assert.AreEqual("!", outer.Read(1)); } - [Test] + [TestMethod] public void TestContext() { SeekableStringReader s = new SeekableStringReader("abcdefghijklmnopqrstuvwxyz"); diff --git a/dotnet_core/Serpent/Tests/Tests.csproj b/dotnet_core/Serpent/Tests/Tests.csproj new file mode 100644 index 0000000..f9cd70c --- /dev/null +++ b/dotnet_core/Serpent/Tests/Tests.csproj @@ -0,0 +1,20 @@ + + + netcoreapp2.0 + false + 6 + + + + + + + + + + + + PreserveNewest + + + \ No newline at end of file diff --git a/dotnet/Serpent.Test/testserpent.utf8.bin b/dotnet_core/Serpent/Tests/testserpent.utf8.bin similarity index 100% rename from dotnet/Serpent.Test/testserpent.utf8.bin rename to dotnet_core/Serpent/Tests/testserpent.utf8.bin From bf5529e52dc9f11ae7ea8de74c149f89509991aa Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Wed, 24 Jan 2018 22:32:45 +0100 Subject: [PATCH 123/230] removed old dotnet --- .../Serpent.Test/Properties/AssemblyInfo.cs | 31 ------- dotnet/Serpent.Test/Serpent.Test.csproj | 81 ------------------ dotnet/Serpent.Test/app.config | 3 - dotnet/Serpent.sln | 34 -------- dotnet/Serpent/Properties/AssemblyInfo.cs | 36 -------- dotnet/Serpent/Serpent.csproj | 72 ---------------- dotnet/Serpent/Serpent.nuspec | 32 ------- dotnet/build-dotnet-microsoft.cmd | 6 -- dotnet/build-dotnet-mono.sh | 12 --- dotnet/lib/nunit.framework.dll | Bin 147456 -> 0 bytes dotnet/nuget-pack.cmd | 7 -- dotnet/nuget-pack.sh | 6 -- dotnet/test-dotnet.cmd | 6 -- dotnet/test-dotnet.sh | 6 -- .../Razorvine.Serpent.csproj | 21 ++++- dotnet_core/nuget-pack.sh | 6 ++ dotnet_core/test.sh | 3 + 17 files changed, 29 insertions(+), 333 deletions(-) delete mode 100644 dotnet/Serpent.Test/Properties/AssemblyInfo.cs delete mode 100644 dotnet/Serpent.Test/Serpent.Test.csproj delete mode 100644 dotnet/Serpent.Test/app.config delete mode 100644 dotnet/Serpent.sln delete mode 100644 dotnet/Serpent/Properties/AssemblyInfo.cs delete mode 100644 dotnet/Serpent/Serpent.csproj delete mode 100644 dotnet/Serpent/Serpent.nuspec delete mode 100644 dotnet/build-dotnet-microsoft.cmd delete mode 100755 dotnet/build-dotnet-mono.sh delete mode 100755 dotnet/lib/nunit.framework.dll delete mode 100644 dotnet/nuget-pack.cmd delete mode 100755 dotnet/nuget-pack.sh delete mode 100644 dotnet/test-dotnet.cmd delete mode 100755 dotnet/test-dotnet.sh create mode 100755 dotnet_core/nuget-pack.sh create mode 100755 dotnet_core/test.sh diff --git a/dotnet/Serpent.Test/Properties/AssemblyInfo.cs b/dotnet/Serpent.Test/Properties/AssemblyInfo.cs deleted file mode 100644 index 2e6d9da..0000000 --- a/dotnet/Serpent.Test/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,31 +0,0 @@ -#region Using directives - -using System; -using System.Reflection; -using System.Runtime.InteropServices; - -#endregion - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("Serpent.Test")] -[assembly: AssemblyDescription("")] -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("")] -[assembly: AssemblyProduct("Serpent.Test")] -[assembly: AssemblyCopyright("Copyright Irmen de Jong")] -[assembly: AssemblyTrademark("")] -[assembly: AssemblyCulture("")] - -// This sets the default COM visibility of types in the assembly to invisible. -// If you need to expose a type to COM, use [ComVisible(true)] on that type. -[assembly: ComVisible(false)] - -// The assembly version has following format : -// -// Major.Minor.Build.Revision -// -// You can specify all the values or you can use the default the Revision and -// Build Numbers by using the '*' as shown below: -[assembly: AssemblyVersion("1.0.*")] diff --git a/dotnet/Serpent.Test/Serpent.Test.csproj b/dotnet/Serpent.Test/Serpent.Test.csproj deleted file mode 100644 index 5d2c05f..0000000 --- a/dotnet/Serpent.Test/Serpent.Test.csproj +++ /dev/null @@ -1,81 +0,0 @@ - - - - {44F33161-24F6-41AC-8097-0AF7B43F3786} - Debug - x86 - Library - Razorvine.Serpent.Test - Razorvine.Serpent.Test - Properties - OnBuildSuccess - False - False - 4 - false - 8.0.30703 - 2.0 - v4.5 - - - x86 - - - bin\Debug\ - true - Full - False - True - DEBUG;TRACE - - - bin\Release\ - false - PdbOnly - True - False - TRACE - - - AnyCPU - False - Auto - 4194304 - 4096 - - - none - - - none - - - - ..\lib\nunit.framework.dll - - - - - - - - - - - - - - - - - Always - - - - - {5A8E1B96-8C48-4A35-A3C4-0844486B2D47} - Serpent - - - - diff --git a/dotnet/Serpent.Test/app.config b/dotnet/Serpent.Test/app.config deleted file mode 100644 index a036389..0000000 --- a/dotnet/Serpent.Test/app.config +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/dotnet/Serpent.sln b/dotnet/Serpent.sln deleted file mode 100644 index d5f33b7..0000000 --- a/dotnet/Serpent.sln +++ /dev/null @@ -1,34 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 11.00 -# Visual Studio 2010 -# SharpDevelop 5.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serpent", "Serpent\Serpent.csproj", "{5A8E1B96-8C48-4A35-A3C4-0844486B2D47}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serpent.Test", "Serpent.Test\Serpent.Test.csproj", "{44F33161-24F6-41AC-8097-0AF7B43F3786}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|x86 = Debug|x86 - Debug|Any CPU = Debug|Any CPU - Release|x86 = Release|x86 - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {5A8E1B96-8C48-4A35-A3C4-0844486B2D47}.Debug|x86.ActiveCfg = Debug|x86 - {5A8E1B96-8C48-4A35-A3C4-0844486B2D47}.Debug|x86.Build.0 = Debug|x86 - {5A8E1B96-8C48-4A35-A3C4-0844486B2D47}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {5A8E1B96-8C48-4A35-A3C4-0844486B2D47}.Debug|Any CPU.Build.0 = Debug|Any CPU - {5A8E1B96-8C48-4A35-A3C4-0844486B2D47}.Release|x86.ActiveCfg = Release|x86 - {5A8E1B96-8C48-4A35-A3C4-0844486B2D47}.Release|x86.Build.0 = Release|x86 - {5A8E1B96-8C48-4A35-A3C4-0844486B2D47}.Release|Any CPU.ActiveCfg = Release|Any CPU - {5A8E1B96-8C48-4A35-A3C4-0844486B2D47}.Release|Any CPU.Build.0 = Release|Any CPU - {44F33161-24F6-41AC-8097-0AF7B43F3786}.Debug|x86.ActiveCfg = Debug|x86 - {44F33161-24F6-41AC-8097-0AF7B43F3786}.Debug|x86.Build.0 = Debug|x86 - {44F33161-24F6-41AC-8097-0AF7B43F3786}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {44F33161-24F6-41AC-8097-0AF7B43F3786}.Debug|Any CPU.Build.0 = Debug|Any CPU - {44F33161-24F6-41AC-8097-0AF7B43F3786}.Release|x86.ActiveCfg = Release|x86 - {44F33161-24F6-41AC-8097-0AF7B43F3786}.Release|x86.Build.0 = Release|x86 - {44F33161-24F6-41AC-8097-0AF7B43F3786}.Release|Any CPU.ActiveCfg = Release|Any CPU - {44F33161-24F6-41AC-8097-0AF7B43F3786}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection -EndGlobal diff --git a/dotnet/Serpent/Properties/AssemblyInfo.cs b/dotnet/Serpent/Properties/AssemblyInfo.cs deleted file mode 100644 index ec8bb83..0000000 --- a/dotnet/Serpent/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,36 +0,0 @@ -#region Using directives - -using System; -using System.Reflection; -using System.Runtime.InteropServices; - -#endregion - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("Razorvine.Serpent")] -[assembly: AssemblyDescription(@"Serpent Python literal expression serialization. - -Serpent provides Python ast.literal_eval() compatible object tree serialization. -It serializes an object tree into bytes that can be transferred to Python and -decoded there (using ast.literal_eval()). It can ofcourse also deserialize -such a Python expression itself, back into the equivalent .NET datatypes. - -More info for the Python version is on Pypi: https://pypi.python.org/pypi/serpent -")] -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("Irmen de Jong")] -[assembly: AssemblyProduct("Serpent Python literal expression serialization")] -[assembly: AssemblyCopyright("Copyright Irmen de Jong")] -[assembly: AssemblyTrademark("")] -[assembly: AssemblyCulture("")] - -// The assembly version has following format : -// -// Major.Minor.Build.Revision -// -// You can specify all the values or you can use the default the Revision and -// Build Numbers by using the '*' as shown below: -[assembly: AssemblyVersion("1.18.0.*")] -[assembly: AssemblyFileVersion("1.18.0.0")] diff --git a/dotnet/Serpent/Serpent.csproj b/dotnet/Serpent/Serpent.csproj deleted file mode 100644 index ad954e3..0000000 --- a/dotnet/Serpent/Serpent.csproj +++ /dev/null @@ -1,72 +0,0 @@ - - - - {5A8E1B96-8C48-4A35-A3C4-0844486B2D47} - Debug - x86 - Library - Razorvine.Serpent - Razorvine.Serpent - Properties - False - False - 4 - false - C:\Users\Irmen\AppData\Roaming\ICSharpCode/SharpDevelop4\Settings.SourceAnalysis - 8.0.30703 - 2.0 - v4.5 - - - x86 - - - bin\Debug\ - true - Full - False - True - DEBUG;TRACE - - - bin\Release\ - false - PdbOnly - True - False - TRACE - - - AnyCPU - False - Auto - 4194304 - 4096 - - - none - - - none - - - - - - - - - - - - - - - - - - - - - - diff --git a/dotnet/Serpent/Serpent.nuspec b/dotnet/Serpent/Serpent.nuspec deleted file mode 100644 index 076c7af..0000000 --- a/dotnet/Serpent/Serpent.nuspec +++ /dev/null @@ -1,32 +0,0 @@ - - - - Razorvine.Serpent - 1.18.0.0 - Razorvine.Serpent - Irmen de Jong - Irmen de Jong - http://opensource.org/licenses/MIT - https://github.com/irmen/Serpent - false - Serpent Python literal expression serialization. - -Serpent provides Python ast.literal_eval() compatible object tree serialization. -It serializes an object tree into bytes that can be transferred to Python and -decoded there (using ast.literal_eval()). It can ofcourse also deserialize -such a Python expression itself, back into the equivalent .NET datatypes. - -More info for the Python version is on Pypi: https://pypi.python.org/pypi/serpent - Serpent Python literal expression serialization - -CRITICAL FIX: rewrote serialization and parsing of strings containing chars above 255, this didn't work. Now mimics python's repr(str) form as closely as possible. -Added Serializer.MaximumLevel to avoid too deep recursion resulting in stack overflow errors. -Serializer.MaximumLevel decreased to 500. - - Copyright 2017, 2018 - serialization python pyro - - - - - diff --git a/dotnet/build-dotnet-microsoft.cmd b/dotnet/build-dotnet-microsoft.cmd deleted file mode 100644 index 0b342ca..0000000 --- a/dotnet/build-dotnet-microsoft.cmd +++ /dev/null @@ -1,6 +0,0 @@ -echo "Compiling .net source (with microsoft windows sdk msbuild)" -msbuild /verbosity:minimal /p:Platform="Any CPU" /p:Configuration="Release" Serpent.sln /t:Rebuild - -if not exist build mkdir build -copy Serpent\bin\Release\*.dll build\ -copy Serpent\bin\Release\*.pdb build\ diff --git a/dotnet/build-dotnet-mono.sh b/dotnet/build-dotnet-mono.sh deleted file mode 100755 index 01a945e..0000000 --- a/dotnet/build-dotnet-mono.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh -echo "Compiling .net source" -if [ -d Serpent/bin ]; then - rm -r Serpent/bin -fi -if [ -d Serpent.Test/bin ]; then - rm -r Serpent.Test/bin -fi -xbuild /verbosity:minimal /property:Configuration=Release /property:Platform="Any CPU" Serpent.sln - -mkdir -p build -cp Serpent/bin/Release/*.dll build diff --git a/dotnet/lib/nunit.framework.dll b/dotnet/lib/nunit.framework.dll deleted file mode 100755 index 215767d2fb38837c02dffec220b3ce69b4859260..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 147456 zcmeFacbr{Sxjw$m-lxr(l4+Tl6fzT1*qJkv0wGL73jrh{0YXVg=s}7J<={C9BE!Uh zNH3x&9U%}Y0yeldL;*nokqCl8F&DfDqM%#>1-*vv^Stlc^~{8b-22DxM?RT#_FLDx z-u14s*Is+=eGd7oag8x){Js0GG2g~7|IL$fi7cEg5X~s@Z110X7b8oPg@DN)v@JbS;fwf z2_i>LE?3GQ1xjp#jl$D~-^jn&#>}2vKK8^C7{ylDEPPX|_~t~!KD_+U10QzP{|1?~Q$}bM>=@z5nvG`^H79GHXBgmk~?n-FNK~w|4A%#kOyanR{km&nLe) z<@u|}zOj8>%^fpG>~QW)j}6=Z%Qx-ba@y|M^GW`Od~G z_ddk&nv5x+Fm9Y+a+_ulYklTpS((PX>FXlB9hM}o}V0r1kaq%6@Q+wJq3 z?o<6s#~r}3LK>y(VD_jcsE8_6OrR5MJG6%~qH_ftqZ$+$s-cf!WuPXJnMfHz)waUX z5gtO-JXn&5@DPgdVBLxK)y8-TMR=@F>tj5Gx)%p-DWkql8E619?heIN`Iu-aTT;WY zDh-NB_=d=29Yt$FR;Eydq)&+teM+dhoQU5D520!vgm#37P&cAwD`T?AvwO1RdSmSM z5a>&s$_NG!8bCU5Vm~1_T^b1r)U4eM&qX_)g_Jrtz$v2*pHeO&@g`Y+`N+KnLoSUX z=Qyr-!Rxi6ep_dGMx!2bERQ!`s80LEXo~1g+T69-0rWVCQ}T5s`8pze?dcdV*GywN zM+-FeQt0t}<->xAs&&g}X*0nnUXzq=2J&BauksIJSeVst`leeu}r zdYG%?VZl1{s~DEOE%{YD^0zjSKoRPTQ?u(44mM9@vxg7FvMH!Av#IgTZ?%5RXT2^M z?2SdT*5ulk)aI0O$?{EU8aeBNP4B*|sFrlmG^M*wciN;ny*>>XcK`!T+G!g=&xT<6 zMuMDCvO9ogoOaR!XtQaj?_@?@rk!@njJQafJK^gZi3@LJ#$?fom~BhCDZMA+ z7E=S#jOoE2hBc?3ww1O5XKir?K+)STYL(O}q){M=ef?sdlEw(hIJwf+@Q##OLbR=} z*rFcE=kld-^jyAuNnKv~)NQ>R(Y4KlgX)W>rhA6trrSM4ko0;|4$foHcQwVmm3EhA z4sBmjaI%r?YTddb#&n7M38%#ylXc2l~(ruVnnM(0B(hNMAtuAtIiY*s5w zN~zWrs`dq$kEG2q=*4iD(3?TY@o)B}vkxtZ-Pt9IF|Y5sWO=QVi)2=Z2)9MZqX>01 z7cs2Y)?}8{=X3BFs;~EXPBFwc=)DYv&18P4h12&4@&dGG3-XyaSUy!fVsuDE1C~B7 zHpd`R`ri{Bpn1w% zmJlZB{L#7W1QGIOtX?8T?;MtukkdLqnP^!p??j(hY#|AxMD!4f=&?4Ve~<7G+6>R_^d))VfBy6|>+fp#x8T8X zsOYu0Oa`mzK026nx&u?FBR!ymu)eKLMB5nCxVCXXsTkh!PxbNQQ$m;!w|eLM z!vx~VHH#5@m^KG(+i`nbx0ux-cwk!yq4U_r&0f9gTNwR%8gaFbR~YE=AF~9^J=1Qs zb!`^I3Pw3faBK(q(LwI%rp0U`RPER8V?2a}C)TcrWxL|<7PTwe2hhBjuWW2>=i|j! zgsSbrJW^$)z6qhe`NR6=Z@j)$iQoQOw7tymbx1yhg1*-8p8emdLF63ALv&$?cRYkb zw)`@Ng|>^((FFG&gK@OSIOaI*7tM!6Fjv1mj)@2Cfo;%wsUosf6U5IFPnBRiu_Q7~MqC7ATGXk)R z-!W6t`puZ{ApSF2H<$sZb!4l}&AJ+R<3w(`O6a%>1LSe~nU-@YyE@W4++yl|jI@k4 zG=WY;2$?ZA;g8&)b=Vb?h^m%|Af&j^l9cK!M8q(#kS@-X0uk!&Hm+0J%|V1n>^f}M zvd)9S<$hB6-!Tu$Bfw%Ibp7j`#r7Vx_ocGd@i)q7N@v+ z^OGxI2Dq3hUxlB7*9G~k&RB3?nmu<=s(wtf=>7l<9xxb&6d`}z?`I&pX0Kw zX!6W^b9_%n_#A`#NP}Ak>dW@zRbeqxc@#{QD^LYt-i|@Nm=Wq%RzM*e6syrxMxwIO z8JD)gFNL3B<&P$it!#@QtH|%0Mv5Nq9sZrvd6NB( z*4IZs=eU4dteYVSW_MsVgx3S9H^WXp;nEm}Shjam`RmZ5I0@kL9|2b|Yb%_tZP7J* zK_}`LcDmBfxvTbpvPiYHmEVGp%=@vt;P~aeMZT&i1jJmNqXXy`__{6^q@WpGP(iu1aq-;}(A ze$$mWb?A?B#`O!Ja<)%HM}?{2O}_v4UWtmPeyI~b~4sp zC{A*F5)=+boXHJS0k?$dp!SYpEwd?MHijF=IZmtei{6jL6oa*8oidbrVR!`FCKHMB z4pFSyxufsnAlA_+PQ?a8+0%L@U0i(^plrc3HbbYKKo3Qp;7P;nUT#`wzaqOHXd zz6u2k=yUWcQ6lPQ9Hb)k?`ygpnivXUS$sEEnSS69rek$nLr)jYwcBHcqmb`Mk?&Z^ zcNMy3?X=Jq>$0+N1vk|d_b^TggLwgKcy&$b()PHlZL8bZi)&1B3*dfbS_S32eZo!HbU@xEX2Crvlfl~9L{ez4k5pO3bmJ3NL@=XkdA&W zN-v!Rvg@@WEn!YunCJ+nwuQlYQH#Ybm|7V1qk|-n2Ms=9UveAGwux;lZHXdVeah>UzKODQwiY#TqqhqZ={i%DU@B>Z{Cz3kz75 zx8ZcWA5LH`VOCOT$rQIj+%UxIoLt`f9S8`6mTXfNU0nUf<3VK13F4WJ;MJ(hHRTX> zg3UMO{J6u64c%2?B37P5sf!?GV|8#Ttc_!tSlS~dAC4JsCLbWvH(WQG)6UatmiIfc z^8SY9JsoX)BQ7B54(!XQEdLr@-mQX@Hs_iph_P+x59^7#VrO2;jAd4*pHhCo(T>vX zwHu3hzoq?h>gd3Jqp7x*d3Ve#B_n+yR@lKXv-aA=znW63O(* z;ZgNuvg&cGorW<3`%|;+Rh;t2BROdMM^DA|K=`G7;2`n_ZG=(d(_AkTD09QdOali5 z60|qUpGND|%(sj`vjzO2t?s~qLCzclT(dnKh0ts=t$0-Cx6WMOD<}dQlQc$?ulFCC z<(u-oqoqGCp9z;Vr@Woe$Yh39*%?=-JN?Xn?UAf=?Xxw-mM}}eK+=@Xx0W!3WAFeU zTX;n4R|eH;co!sW`ydWkHQAa{9-eK3ZA-bc{#OO0)rE!R^c;6 zn5c+ww)~lZKP!((rP*>$aRlta{WP9)v%eo$1i@hG#~oM<5CxkKQf;wS;|u1JE35sr zRt56@?Z$Sk1PK!$a)JLDu-eaUZMNQnx9V(@DLBEHUiv~cmgdkIvQnV)Ms!eYYvCA3 za|Rg$F*KUVOy_EJ;u&3E(lP&E5ifOMcjb z|DY#J^#5%^|L-|a&ThXg+U-BT*=`@%vfH=*s4?Svwcn>dYLV9zmZMS!#p(&hWSx(9 zDg~dA!0+k1E`i5bMrS;-Sy-cM~At7UvNQbe(dh_Wr7Hx(uV^2uGAZX}LWi0A;;7}UoR3`HJupl;c zyhAt1c6vC_s}Q3O9yuxiM&6G5cRGwewnyQ|dI$wd-(cieesp!=?Ry@j8Y4uvbPs;L zPUWN}ZfU$y2B<@0%rQIqx;L+`pZ2>YEQTUuS2SQz_RE~vajnHM$3qYLa+ZmimBxWn zOhES*Al%a3p*b`!@YbQPX2JppWZ5?)mXJYJ*%n z=4y6eNa;U-aW|zg;lfz-0Gz=xpx-jvN*TOSpu?cr@)Kw=6;wdsbPQ6FT=-0gKGA%9 z3NzB8JY~tVQ?3hr~9eya6DM0AgWeHi)}BO;5A5zCCN%Hw;tVz81QXh%_oX0_wbfc2U6ch)(6ZN z{VyunxWa)2haw@wD0k`g*lWpyJ>(yF5BiU)>3Z+0_PLdZ z5)2%VFcw)Kxs}IApMoUf7nVMmucpWCdwdZ47%!Ut zdUV}7pnAK6MzU@rb@XGZNY+t3bZFhX9-5k8e3Vb~x3miGa|=-h>Sq2kB*x^~hmcpo zAU_a&o*k6uG4!s=1EC~OpYr@P;DJ!cqwBJmfue1+$+o8o^0(T`J|}I4y?iDT^34eN zR)&0P`;MCD6G5IZBFs~Hc2b@<={x#o<#E+KJ1fttfu7mbJi92*yIG!x14S2C@u)A|!Q3cyPdylqB(LoPJ(Vgw zTf(FD;Z_EziXbn{#wD+rL0+q>^4by}%-uy#*7f~!ZA?!W%B4JEo2d-K?E4ZEXVF(1 z=)1m3pYlZX4Z{4trSBuwzJ|cQFICy6JQ4c_VfO9AtX|676zIF5N}uvX^bNx7>%$Fz z=zBeA9Eg}iyke3sJZ}U%sOf-5+o7(_RQ?*Upc2Ejs&T)!)$>}wgq9HK*>WCC(Z!zU zf;RTiDm|FB3(v2Ec>VnQ;=xo*^lS|DTwbMTOL#CB6Fn~kdcIJlXG?f6R}(!i2710& zrDsccFsBne7Y2GhRiy`W7U8)h;8|0}gSnpYTo&-08R1FG+92kD($1!*)DB8LKNCs^fSWpDV9c@ynQ8 z;gB5XmvOklVc5s(8p~|PnGU@|puO~|jJIxu!)**N<#c6;jp1d?bSoTIw+Ju((BU)2 zFXN$G;gGpScp3Lx8SMi8;jmxuVbzBWp!sE#G0v@UWQ)lPJ|Y`1d_b}?cDWUfSh0Ks zAIUdhh~z8dlw0BG6q6NvL^fcE$jZ3oRyb@0vaD+vtK15QDvQ^cl<`T1CqV~(lQGGy za7c>j8453Bk6Yo;6XRc?vB%-&J@IX@Q}`>>ID&15#4R1?YYgjD#uc~1p(mzuFg$e% zKC1KhYMnC1xD^gVF`a|qsZ;P#oy)6r%6Q{eI3&e%4u+>r!AEss7n(|jWtZ{Et#EjX z>69_at#C++;bk0hD;$O_-sQUqM#dgH^a#OUNCnOB>oFK%eCj_M_twH9TLB4mSMgw? z5#b>e;ptU9$M`(Hev3cW$NMHM#;S!|;)+e_yG&%Yi~Cr=4M_B+WpSDEj`uwvF|<1E z6AA-Vl^tM4{yGX{ANN-5|G33X%5hvE)GoYEr%DbYbB!;9-0x%Ve!pUMUjH8S8MDb_b3N?2O*C_7m+^ZuTXyYsVJRT02sT31jKUTu?s07S&0@z5y@+_y3o zGU~fWd{amt%!SA!%~-e3j=XO|pr^%=-&IGU6#lbUuHUs{v5AT*x4?F-wPJ_GX?<4i z2q?B52eIX2ACxUrRJjj5Si~0MtSt`&lx{?Xb%h@h$5)c+taSZN=>`3%b)8(c^b0uH20VCOc_ypvM18&(KJU#t9#@r%HTv6GSYqpT zXnroVlApgmzu%&06Xtt1CY}Zd+f&CN?MeF%js3LjLY0gmJGhP!@2TTye}EAkj=^0Z zpH;HoS%Iq3C$sf6v(^9)9Hta$*-fi)hlf<$Kers%m6%4*9l?_iG;G=< z?bG4=QAWl_JfEjk;k}=(iKe8ix@Xhzeg)sV(H+u0>Q1zk{ip*v4U9sCKBDx ziv4yc(M_3wlf{WF&soqlS!05OA8ko#_88!eO? z4ous)$ZX6^G-e~z4rSx@gV`8ruTpNG#-d4J5a*Ndt_3Dj07(1qN!owlQ1*x0_M?Sx zLv*4fN4|GSE+4EUp~7elhFHlaQKo5T!eTAcUDmmFhpb!vHNwg1XRPRp`u>J z0^Yl%NwJR|OUE<5%ENOp!Q=*7@R9#W{4R>LTX)qMcs8=vy0N@H6vg)o9tImOSw2u= z5M)bu^hf9v2?^qQ5tW1x75?rcZtpr5daUKz$6}7icdFPwN?#bw8Be0a{hd9=&FYI4 zZuw3r7PAq5XijojI)V+ybRUBEVZ^jHwMlntq5-r0;aFDSe^+;liqdWoeNh_f;Oj=(m{^Y<;_`JXM| z`R5k!Y}x{zcej8Cr^nhFlj09ru2eNoeNcASbCFHdcW4u~&iYxW@40_g{T$^@VN@G@ z|7_NgLAGxs1jaX*Z6T?esIEUFQsmGw!4ne;k3WNSKPB(ry*F)CR48xZZXw`{aqZUnz& zB>tkS(9@RCf1=M(%S+3FMyCh;*i<%>>UC2YY~y=X@Ax+Aw6CWj6Pf<0T_Z5(kl9~G zQm+mgP2YnM^tXv-;@d=#*C@)2f-5tUHl)1JW9Vphdgq9ZN}LopW%>~Z?p>z4Kq(^-F)!0toD?$S}H_Kc*yjeGe)a5DHCuvrBh z>#%~AD!8jZ5jUQ~7m*~!>xCL`ic{Uyrwz@mSz1{Yk#Q|+eh%9(apV}3&Y)(7(P~>5 zT2$H^nKqmF#*?u&- zbhiJCNE_1aOC~g?wM`c11(iW$7ahbpiJ9?lv%1;-n^3SX@!i)*J?BB!EanX zC-T~o29vMN=Hx6Jo$-!#8`7&kiQ)wBK4#a}lxDMJ#WAFv4w{B7-irjSJ?GPHz5NyC z!gLC4K@2IVJ+)cBa+A%LmceIrIRSD4)aPrkc{ZKx$#RE23N0IHXW$H-;@B*9^B)a? zoc*=CY~40&5YDtO;kzXHDcCEFNtsAJfocuCGK9{wwT z7ENIdd(LvYfGc*3ia}d`npcc~a%~;1SgWY6U=A(w zU40q2ry;tGf!ac&Pvh0JajpzHy&y*iAOH0_vd9vs=&XZo%-<0MmC>l=VEz=rj~N1w zB~slk#`#u+9}B31;*az3m}_r?Yi^bu1>NZS+yQiecD>b=y)`;N=_lImw?*)Xby2C+ zBE@-f3(8?-h=dGQ5!DPS=k`85+)x|Yf7{a@q9glkxur1TtD`on17q;NvWGWduiC@Q zRciW${=rgXLHvW|AlVeivf@79pTDKO6(q>=ReWnL7K*g^Q7tEm7V5z5Y?TgDW7~80 z1|s72z?hRn4g5N|9+G2v7DV(QU#|15#V_V?P1uOGX*h0Lo#vGNHO|0UsKMrxtG}lI zY=E7e$J8~a{LA_-h_7i@($f5qEqM8>34cP*A9v(Y1C&uF6nWZ0_1HQ$Vo7b9rz?a1&b-(F0V#a{V^y`Qo6%CG4^m-g-|_S(z(u@L7! z+Du&3X2N1KFG9wYGDhqsUF_xs6rIDLl>999aOYiZw-X(Jes z4aPC8lUUg_uOu%9gh>5CuQ<~5v-*;}=q%!T#dy%)GJW@gzk+*<0^ZWYcu}Zpt1s5$ zwJzLw6zXc)m+%^+w!W45GS0O_gDB{mM7hpGOBlur6EUYNPDWp&%e|6SO|G*?i8)N= zWF}a`KrioWV)loLpU$dq#v9-Nd?43(xUv`Pa^?N#(cQ>hgIQ8N)C$u}Vt!6v@0bKK z=9#7`qvgiqY;&BH8wIBRSqA{4MdKlnc-DyyDp$fBqfnRU^t1D@se3nN_QZ668yn3d zL;M{G{4)Gqg+GkE<`(>+Uoog2WZrq$mi)1IJ{CM;fSRcvyaiAIM0T3K(4uEek)`oG- zgH}*4ID7zeyo;%|E>~adFnDti{Wac0%-6V0>BFx_a%dyYgN{bhukvl~^#}qDACx6+ zu7UpL)bAFwIoN21QKL+D^+jygZP2^4kW*kcdJYqNBGXIt6%R1?B$yopdOKKblZna} z$Ax(`=A3j1b6v|<%;#_<5OdIQ5B+ci>_pRt9~Mg1uKswK9phaC4AQW{DejVvSd81m zrFHp?+`oWY*Otk39J*8_>KtcbI`1vS%xZtv4KBO2ei`m7BrnZMQV+B5LLt-ZIz^mR z;{m7zfTPD;0qWBA#jytO^0A@i^#=_S?X5RcgMOkHHL;|=1`mk@_q00D71cgp7JQlE zNO-L1Mq#BdpV`l96Jn4iV<%woR@Rp>cFRk`*;->rhi$Qwtqtvjnm zN72l9!0}IdMGtRpRhB!ea03^LlW&o;YHkv@o3rY$B(B$4H9Ltz;SNvY!osl~9}Oq! zJV8&D#p~api*%OI;dqD!ydTLl?Jb3y3w=G1Q_tab*N6uLk<}u?H0>QCBJaQt>4mbE zIl^D9sx7ZW4eLut>LaQPeWmI~INqm39nNZzh}d5 zCurPm3x=LrSI$;J2q1K^aFlDW?HfbidYG{nda*5pY`sx7K3^SXh8fqxs8guz`?LC$ zPeOWzul_!qJPsK7=dW8pqA4vTIb>CykL+*jqP}L&G?+8^<4xlf{9&{2NR?kGN`B^%v z!RtmowZ+Er@CH2lI~a*R(ay>H(Rc^PoT$Fs4$Sl=e`CJH8EWw5xj@sGB-57xTj)y^ z7ry*GiU|WgAYV>}Hr!*FK9og#shJ=$#Fv;F6i)Pu%f@A$aGbL>IE=Ir-;p9&T%eAm zVCYD;F$nx*&LB=iRt0a;P)m zTL;kQwgwm!b%h@I&8h1MRO9rF>ntr|bKTNfoV+Y&fv1b*P($%J2O@G70!8W~6x62o zc__sX)|v|#s82o7IrT&>WPQ!hg~ae!)g&-nvM#>^doe}z%b2Lkd7uUj8>{B1OL*Ha zAX}UikoO!?iP&DEchdGX*5Z*gUj&N>1?6IOOU)@^M+lwh*kZijx_Sm?q3QVLsH!W@ z3H#O}p+R4S;i#*d!lJNCMp3q)sy-enC?3m#v+O}lsb;o2?+U2I52o_}eobjSehc<~ zJS=0PuI93Y=x(KiS0UFrsW}@JY}hPdys8l01;46}V8KMLc%Wadpr#c6Z`Ks&SUaMB zEwc=cw+P)#Pu`VZ9U*JW%h+Wj>af0`^E7n6oUg&j$}&o;34#N5gKYjfN9JcuwiK|?aGyREv1U9hn^BZglz%~Wf@nlyz4|{ z{?L;Gh%okAvdwr*z$@>15dof6){<+^>D0jclHf5hh^+K5E_4rhg+8qu16TR;*^Jo@ zzQ(K(fV|cI~=~O@5Afk{^mZRqT9yPu>qPyLj+2k$mT z_Oxd74q(+D$qZ&;nnV_vdafh6-d!~b zKP)K@d&j$yPg+tQ_g1Z=I0t74 zd#9iboI0>jwFtDbl&Lro+h1cO3_g|wE;-WEE_Se;=~e`%Jx(yci*`(X+Hnulm#{)aj8vpAoy}!W6NaS2b>YIr2!T-=#`U*sQ+MUuJ_+7sY zXgPI9G$PD&m=PNgW!OO`GIuh&s{>kE&b&guPJMTQwi?sqbZ<-RrtBH8wt_CQu!g9j z5sH_^&WhET<0{Vpc`!;B)({0A{&+d?NDkcFG7ouwlKMjnb?m?n8g0#M^QCV>MS1Kn ztQ6mkD;!r6Yue=Rb4dv;D)N_jKyDos%3(7#z^I@?*f`(7JSKI820p zu~wGOMo((m`~iKf$jacPCS=+OP1vE{yQ6IXhdu7Bc1v(O>`lwB=7P$aj;++A z-JGmlru8SFb;z2metYFPZita~uWN#%pqq1^GgSB1(4CgOA|jn@tLtNX!D~PFfTD*z zB5MMAI3IR5AU5nIl2ZsCPgPeu8g>fPbZ{c}Ciwdn9N73D%tU`O9yi6QQ1sN$^_$g5_+g`2n~HyHaWfFuhO9r;wH}4Z!S)q)AQum9;d;+)hwTm zwUO0RY`ec0T6*1d5w+y8)Hvw!+xXSmX8-X2egCkXc53~92Srbpm~J=8EqzxYrr!6E z^rAPoS5Qm^g}OlvV`?NToFm-Q_d$`j`qFY4z5{|h*miD0$B6p~w9B;K_F;DsqbT2Z zJ}7SKKFD+L5VlWGL1$Aw9Eq+!rK@OY3YO-Ueh|u)l5$B}RPHFqrAtf^Wnv5VNA>*^ z)~>z`k#X@HG?_wpS?gAQ2;RE1{M5Up`j5u%XZ}p z#=I!4EkSeTylRP8yXW4~x!Axp%&VVia67@Xyock+L1wF@(%I;SG$W3EQ(=Qc8_L%q zZ1m#_K09si&6U3o0={CWx|WM;|P zM@ylp&@;cOQ2rw=8x2x#BcMMAP+hSXGMHcCM9fM475ZCQzhFjnd3j7{S+aamgaX#& z?O}`3>z#;-wG*Q!$K~}PXeH&wz<~(39`?x-x1A7r#_Fc2qV!Mc#q<$ zB_k6!8q*$^Y0Ai+O%53TCK#@@HKD%XcJ$?#?>z<$=z(n5NMC(4=>0;MM_hO(u?3Zh?otD(vajxG|xbx*L zgjKNfh0V!syWpv0zC!4t4So2P_8GRr#&T0*>`kIUvO6!D-GN|#uk4uDcF0S{c2$Tv zWie3SkuvlS3qa=KS0GEZe28qYVeKZ&d^nRE_$i!i_aeJ8uZ;FR?y7<=C14C3U%mqB z!d-mwp@}M(dk$4_? zP^1l;EmG_}@@-Ae5n*}u*@E&c`#&iU58Bi{`t zFc;b0Q~nk`vL5lT-DVMU#@#@)FO$l4zMK6n?Mvb?_Mgj<(`Tao(y0@IvsOE;pF<-K zwqI3;zGL79$T|H?12%prrv*F_n$u7HUDHvD(vu+hZ6N zsXO`|EsUmie)eoAG>J-ek75uJ?!b)@wf%J3V3ot1Q(Z1pWVBpbEZ;s5vD(AJ@HAGc z3(HW;S1xo2A60gnX+IhCvpzEc+t55N%VC zw^pBuB2VnmexIM#kzD6tz|;A5^xHbS2L;f4!f)$5iH}rsM6;{r&HfhHr(hZ0Y5xU( zDo-Qz@(9s^Lr}6{Z2T0;6dfC#eoBeD9=z;DSq0)dVjE>gY~)T)x5Y-JUELcywbQ2r zsw~AGRj&(bpMy`AO96xIBbn=)!h8V4!+Bt_43qrY7q8{MgmX;HP|=6T7s0B|?2OPi zKrTVo^KvoPa=l?0YrYFi;T+YGb8D3DdiVO19^}b;qI&%P>oh%EPvzvZRj#CwlP!|E zf4G@^x12Uqz6g-u9?Oya-yPzQ&3vy7r*HG2iL9L=ae6#s% zh4tX5pUE6Osu#Z0XCSs7NVh#k9$3lL@1~r`pZ<(&wDwv+|E2 zDK5dS{I9?}K%?xmSI5%QmX=35&C0K-r(&(-;%1$B22!d~^t6 zF4&_uRaZSU+s5*K5RTkt3~9oJ;EUWJJjL-r?gbt0=Z^K?(F|Cw`A$Dwfl-h*JDP(m znmYY-om`d>skkR6Q=vFgc`P|fv3v8J;7k=b=N?-25!!=!fGFIHK#(a$!uYGzyuXRs zkHKiO_U{^~g8gHLu%9ZGj)8RQvxKB&k?3Q+h(VgDXHreH?9Ym1tW{V*G z&<9cmtkzYPL2-lm-EIB1{GR4nnz6pqJ{s+e`cPfa>+;d6_n)vYeEcQlp6=^#%ah)Z zJ@=zELj}eQ?e{Oxap5b7OKG{hH^3Oe*9b7@T7|w2&z^!QT~~P?GSzNO@7gE9#7J|( z;Yd`)cd4=E6KCAi)#B|2#UZYpeo_NphfD(f^#^;V%`8*9r>QHz zh5mNyS)tfhmL7t zf|`dA#J6(39{9M30}^Pcf1iMS*vGN&v9e*&RQ55-%8*l>q_WTq626(JitvyU??Bh1 z6;sro&8yz=(3{b^A+y?q=9n+P0ks7l?e#T?2X(d({1*;dXN~2|@K~3JI=X{SNz{?x zq>gxn3}q6oV`;ak-M-rQf2Y;_5OBozxUX!g+-ZGJ2KxT4L4D6VoJBBAzYXp^k4Ul| z?p>gq4~o-IQ%Wz8@>?YA@C5T;AuobK#s`$2V?i^1FQFgwT-LVAU(wA3V^S5pJ*4A< zkL@10p@E;yL?j{DFm!zrHegEs4HhP7HPt~&f0AW$P`ac2+V_DlLrZSu#lhNkm%C{AyjE?uV_WUGwghjEw;9RYm zZ$yI6;dg-s;nOW|FTx8K?SXR1zHD_Q%epz4-@Ol+m)?iW%kM+x_y3yAnPB%`8A4`Q zyuM{9$H1|=d<9~64ePm_X~PvOzFgrIYdAia!b4d^)oMoeEL`#!7XHetw3uqOodQ?f z($b+iJWP`%KlQ@p6`Q}s1M{*!%@Y5-U7w6br8M2tu3&%O7F1tp$&c& z0x(lQE&Ey&_>%=VQ~ST=if-^^pMye~%UJ)b??r_a+Sc$#zY#M_aqUY4hj%>7MHIHu zipV$}z=Z+8=RJo8^ueL+4tL_&3q+)y-fVGyrf$TzMfTb}4s0|!43XE$QaVk;Vs_pVyKBD3vA`6htm zgW9(K4gKO6(LXt^U%`J{e|up6w77l+|84yv1O3zE`W3`GzHV!)RvBOTb<;k;eq;yK z%jXzJy24WMwxJHZnRh#8Em$?(3I`iiKFxUoHj>~bCTsMKLJg0ej-Epk%z8E&} zFapW-@NsMe>d>B-YRK3+k$of+Q4gPIJ+xx>8*imr3QlBBo;7GcNIO70qGqH`D7;K z;j`JxY2SLxnDz5QeCoY7@>@rYTC3w*gU^fL^=eUfmgvtNMUP{1iP1G5DJvgc>SKC2 zXr7q`g@S(S;?m6Eyxv^jfgE2(4aMip^Vy&;<|iIqpYWresiO5DCc&a08^}8{FOJe9 z>P7$7zF$Hg#>y6TB+JJ+N5bcpUowueOa5I$@rxlLJC1Fw`~{QRpp66wzj+*BIcyzC z^d?%;`)OaXRu|+R$NDD3ut5tAu&)i;#syoYf!GgLgM2)sv^$2%XelDthz%h&YD0*9 zUm8$j-XEhb2oQeGJ|f7}M+Er=h#^Jj?OoUmUyo)L$NIuCY>+9jn2L_7)(~Q&8Uk#f z0anUss8O>O;mi3PF%jHPgmps>K}1D}2t;6~*2ug;>r}A5aE$i@)IcjYB%7rm{G5G` zXr>|}=kNy}f`JjNFC4=L8a{%%BG1Jf!5^eBIQtl1GQb2qWiWDK@W_?J*^zJ=;)D) zyzdDrY%rRycMw`k=UnuJj_h}pd#RYtBrU2lL5u0cEgW<*t=5^OMRg`wCH47tuslB>P*lAomf`J^a^*Q-I!>Z?im8MMj@~&4gJ=zl=m0hHJ%ue zzzikC`m9l8ATKR476c*2H#NWn2A*YuNP8uTq|!dohBG9CuX8AdSQd;8n*}ecvlx^i zn#EuYu`DFGq;HittIlFjhG-UpF~qVsEGz{ttFsuCA)3Wt46!UmhgtBlI*UOWqFD^a z5X)j-m<2DZvlx^in#EuYu`FaVrFFr}>MRCjh-NVuLy$$-YsW%0f%etK(SZ_a3xADv znjyv)4KRUJsBGV8D@mYz`8YaIax!L%JHs#9e|vw68Zj8nS2qYPrjr5eYA;)*Gf9i; zOweLF+3s8&4XSh|X;GaCT1@9*qLXQL*^{)W&IB!{bF}DWTCFoli|S0!VmcXwx_(%# zGf9i;OweLF*>>EEqy<&^Gf9i;Owa5N2D zmChtBsxv{0>5N2DmChtBsxv{0={!tofvH!eGf9i;OweLFM~hCT)jE^3sLli}rZW;r zRdyz6QJo1|OlKsLs&po4QJo1|pi`nNrYXeu@&P80N<`Azn~6Z7`I-iii|LF+QkBjm zEvhp?i|LF+QkBjmEvhp?i|ITpjKC_LNm^8Af)>*miKHr>Nm^8Af)>*miKHr>Nm^8A zf)>*miKHr>Nm^8Af)?nM=!$6yF}{3&38WH{^pDL%pwN6xgUH2nMk1+7XOb4xnV`jV zMk1+7XOb4xnV`jVMk1+7XOb4xnV`jVMk1+7XOb4xnV`jVMk1+7XOb4xnV`jVMk1+7 zXOb4xnV?y))HR7@ykF@e+=eUOSrH+Y2Nvk}PnzzhQZ zBTz1MrIXLUECv$#{L7c{pO1wSHTFqXD4`f%LLq#1Gkhi$lqQb#t&3rU0^lh9w0$PU zXAE^%cZs=$CSjKhjla=G6HKG7M%b?#@u3Uq~;2>spG5!;?6(TbkY^|PN zqvvg{yoC^ubzqSTR?|YclOl3@&Glq#g$S44xOqhs*@$Cxrmd+J5uf98q`kh_(v6a( zozjxP?xRBi7v%|?U&ITiM!lebD0&^X1&R*v6-glnG4Kr0f>xwO5VoY1%7fd_D3AXz z_f#L9md{^_f4cS2HO#zmSjVfVNNdz~7~|HQVd^30Cl#kvS7@hZ6^CynUQBdEFWN=_fN z-qwiTg&&-~wx3sN3S%D;rT1P&T~d7*S3I8x_OS*h@322jNnng* z>{a-RtG68z8B+(LZNC}XjLpz?2x$&7vhT}Eeva4Y6M^UWPWtwNJ&rdsMpt`0?!R?BxairxCoCaVGKlzDNbtpiZ^+vre!PKA_Kt(e*vi9YyaG z5xt7vN${MQN;L)c;V}~Ic(sZjiyy#09^^p(uspj8{>cbl@3m(WuQv-e4ivwe;ve<} z{JEo)(+P{{rlhRkBeDTQL>7yJ^wsNr(D;79 z;ui6|9kPDVn3+Vp+=vJ|@EhikW=v43#uc!<|Ow-IGl*T@)Fm5 z=>GYV3+Eh{LG~)j$kDz9yk14BoT`JfP4@E7Ho=bc7Kpo@)8%vyIhhwP638j4&E=WHKp|!9-J|UiHl(KI7=FR6%XRc+Rqo+ zGdhQxB`3+YX6Z9Hf$62atA9W{aVx4W;oKd4pt)LTKLpJw{SCyG|A8X=Z*5|V)UsM= zl~M59ghC1TrFhv~4jIvLkbT@W;=|HApm`(w#k*Y`uLIZK-zk9?SNM90e+`)TGjOoE zma^`}T)V|5t%~(HMXK}Px@1n}PurcX$V)$?s0=cM9=3QKLQG zw>CHMBZO2DK`0k|Pc$v>0OTTamKlf5xofi#zAosG@#)u)m9IS=;~k&m)o*x*yd9Yc zzrI6C-RR_KVK7Fp??t~t{6k5+zDL*pa1!q-|BsV+NAW)y0{_Sm_(zj?o9|DDz(1bE z+kAf(;$1Tz<@%D?@>{_QX0^$QiI{i#T8O~*H6d8AbPIlS{=u`iQ3>lM0MtJC0JhbSbK04E9 zWDN4X6<^opXms{aa&&l<)*9&a7Q$C<2@~YPMCD-L9@;OyLH`PC;VCVl_Wz;%mPfZE z#u%xqnvcUpTw%PN?L1I0W^ZVZrMUyF`-nYJKeZ1qV0V2D|wmhA*Wu3|rUq0TK zU3VWRK%6zo{Amu#`(@$tPJs7nb9?610~pLPZ?=D^ZfRv~Y5XhlLwj{NKqlW&D5gh< zx+lZH@Jdi+aMIq#tbPTa8F(9s0iuozkEhNXr%*^IY83+sCO;HNQ_)?dud+t4-+AP!HN{s2)e2nIYk zzNo>TmxNM|tU)Q~Qi)S?XgFG#ScZJkJOjnac-M=ie1P*BRJr!-F2a9Gz<-Xw-6&aU zZ))z2T(WFSP^qj}6He}$oYk)fX7KO zf^b|Y9P^?a!yI=V5#ATcQ9LpFElXrUG8WKN-W5XLN55VY=qGs+$yW+_vh{_F`q6R> zb5{Q~bTy94S&e(qpk{ck5au`LR{FRNw`_3cbojBpIA4!_tuwg^_{4A8-tkR0I7f6s zLB9?wA7XNfy-v1$a;|(X8UYUQy*OwKPG0L2M{?^eHxB*oy=~)Yd+;qz#^xP2k$Ifo zz5|;N?zmw!{tJ2-!sahQRTn2@BiSMz`Jq=EtmV%Kxpv!o^?GyY$7HIHwS|nq4s&yx z#8D}&u3;wmrhIR6ER#M@;%OO-DP-j~wzjktwulSg0uHk9lp{YQ-|ZpeM6xxbX7tfZJvRGvqZCY!WsKm9kF$V zquN`J6l!a%nI~7I0vPaWq(v$V1}j2SHZtt7Z|)8 zkoe5>WFEf>3wWYA*jt@!`9|`1yKiGH0$#r!cK-IGtt_e~L-=#h3cPC3u90O$R3swJOR($PY{2C644 zwL9fTa4G7ds}Ltzr~c?6HvTbc<6Ti3S%Q!XQ{*UEJMwg&1sfiNJ34!1xO10uENx#> zsiy;ei~5myv5R$jWGnwV_g(V}vMN6dnKg7w`3+t%1N9RoAcZv?Im`7ULEz|B9>FV) zTjg=Q;)qo~7bjG%;rLUYLL9?cjzwIMZSU7{xE;EW<3ugVWaBUle(=A zzU7%LgV4iudGA)goc=Ts`OP_S?>Q;9#{1NJj2X8Y z>!kSYn1$;DSC*Hbe0+r*ccZc|!o+*rq65urv}JU2X53zj=E28iA>gy&;c;_Ls`@epid?%>MXeSqk`L3O4-9nw#sdtIwLdB>kqO_e%ORNq<|v0aT~q&q%jv zsB6fY?Hb6xo22t4{jQ|nYb4LBk`|iYZp@lZ&BXMzZqu4I+qO|^*EVAIZ=;?E<$6pz z@kOKsb6-30i#mw;{#MNQH(Qbak6pxcZq2m3HSs%+uN&WBP9MK)eAb*b;n)dTbGf9; zCz7XU8>USjIgj+H;cSogWV)Fi+e7MOy-fc(h3UAdEmN~*-c(YLk#s=PFG~7VNq-{g ze@VJY()MZO-&xW#rX4mdYp#{_(P`BATS?zQS}+s0&25`CGq)vvuTG|i%k?pmu8{N+ zNk1>?Rg!*d+qIC|AlJ`Jn%<7G1xcGE?Ur;qDa+jLXwMnjvD6oD$5P)b)b*0SB-GaF z`RQ4+r|?gmPHl6hvvy9G>$THA4eGZgeNfWo?a4Dj(k@Bw-=4Wln!)n+&mi?#Nq5|V z`R*<0&5}MY>7OO-=p*OYKFUt+Bj^5othJ8;Uof{doYPk@>-x^=L!8Vc=Uk)(bJEO% zXBNykNVDdenUwmfq<^2e0hkfJZ?jq1IA&{QI~!xBqJ8?%Y=IoCbdYwSg-CBS$6F|C za2tg$noUK^aAkRX33$%lU=5u}1%mZW~w?N9U$kO%`&gB+5P&hv!oMUl9JahPr(8ijh z1iC_?$>uD9zADf(oH$xUIg?LGsiq2w6}$}2kxk7Sw4M^yB@Xhs6g!&dQ6~+7W%0`GXUKS&c{W|9u`_J zv_*hkM1B2SV&bD|#>71Hw9rlw+CG3-h5?b=$7~SjbAaB&TF!5T^XmfbYhD)UF+gu) zE#nUYyyocZ42Em&_fn_P@pF*^pHTm1GF%8y!o-{ z{0pE3fF2QAK6j3L3ZO>?8ZOYs%;N%WBhYg5gviYRv>>(I{9I^z0$Ky;8G-h<&<24H z19UIGlJP5nN`QW1PBOm}=uDxN%xeN&CD6yse+zV*h2E0y z^hnN_8&a#y80XWdl@|p1l<5-M-vMRuy}WS(vEH+0jhP@&D`rAjM9wyt?lc-N&m7J& zlbjaxn7e>B9MDvO76|kiK;%3OoIR=YEL0Zge6yXJ z#5)P?Q9$*876|l|Ko^@u0zGG;qXqhdg^qRD5B@#Re(+**ywI9zsO4hwF^3V!wpowU zw>~BLZ3Ei9=%dyM=L|q^r>-*>3N+6`mk4yIg)S3lmB?LZE*I!RKtDl`b%j7zS?C&p z?y%7H0{zfJHwyG~3*9WxOMvdh^S4_BdRJ&SnmYt)F3_JhnXd~p9?%}Cb>?n?b_Y~% z)|m$cS`6r3tk^#+(D4G@VjdCbY=LeyKa;XwFVJn~DWQE;ps$#x1$xLr8w7d*&}`=` z=GOwfWuc7%6>63Cfs3Mydkt>0M(ms z0AduJES%rMDg$F_0MLofw@p@{%K)7Ms79dMEUi|c2Q1Ve(624j)FPBVU81MFM*ymD}-~Aa6V&Bk{p%+sxiMdrwMd|Kra%4 zmQ#iE6$|wP+K_t1oF%l2h4zlQA>*Pa`bHn~d&hhw0{z|GnMs=i8oDqFR*+`1*ECEw zi`U%LumjRB)iJ$IuD^;jWq#1WdD72?|7A)4Civ7aYN(fVuSQ;LdX!v$EQ0xQV+Awd zuBMy8Gok5wNT)VUM><2&S(5J3^aEgaN9x-%7o3aNERb}ONG)sfZBCifnyKxq<_=WH zWzD0I-qbv}oh)O@e5aY|1H$=JN!8k40zcN=+`{y$E!04addAO1rTzpSpZbrM>8Yqx ztJ$x$Bb7FbkfzLGtGC31^>y`C7|9pd_Q>OyTk9c*D3SJ@E_v(so|8R zwQ2JlsA=yZGQ@6pBlag`V385+xErvnQgTB5=pOV zqn93UI}do3YHjtLV9F7t$Mh9A6!4&el${+A}uMr zF_P5(LYg+OkDLz6%_!FFLy|rr>21A%JR0h zz2;zTp!2M3*6h-W=K^L<=Vx%eS0^IS?Av)Bt`F?I5d3OMGM#3Iby5EtU9`;|%d~wg zeLD&1G_%9l>F5ES8GjnlWX_%O&k@-`aW&A&c#kXG@NB@6!TnfXfpeIuOnUVz1iSP?LM+=*fjHdtpECLZKxKl z?N{3}`;&NEX4~9<%?^{8&OwS&Pa@AjlWrT9GKWmM3#no>UDZv$UD93Om@;4PCjSp5 zeWn{VV*A6!i5-oL*JO)Knbl=N*$vpuxDxrcEzeDc${iD0_E z3&Wi`r{l;@Wp&LMJG>12U4mPVMf1L%BwJK{y<%s<(9309{5Cy?_1HBIn! z5}Fu;b_}2^LDPQ^XF8g$!^+VDp=o}j%|QX@`FJpTMgZN4wXa(P=q{`yZ3v)y%+}^r zfgU#0InG?Ljc?19X1uvAfF8h_*)sw32v*V39ywL6XudBHTh)5(-27<(J%aC1bxrbh zK7+P=S^)h9>pZ^?pck=rvro6rc_miKjtHP}W=C_1LititkJ-tb7oqKBu8KfAo7)4( z!+P2`V$gSE&;v1ud80&`OPl@(IsNx==4wE@`cVDNO@A@7&7&gsaEhF>&Epm#=Wb@b zLN(-^WByAZt)n^SRe{dQv!9rQovT!PPX60X;{Y`%B=Na>kl#<5HklB52Dc<3^g>fE z05%N;gNfG{&Hvp%j{)- zA<#=TEW_UBRSU5Ud*hoMTK4Cg&j+o;LVs*_Q}fMKfnKU%`R1FsmX>ZA189wfT3ULr z>-0(sb+wEGbhDMCCl;7{EseeD0<*!==$C!W>z2mebRT?1U2S1+x~~~+A@-*G8qY%P zG50gGEyNykKeIrfmul$Gh2|JbV{I-pr&t<2zrQ)p(t2U%{^oKE&1%V|4ls9EXwQ~$ zfWB?zSStsb^_I2}v;)m^mc}|d$h>K3hk=Htd6Tsa?D-a%4hym8TV%!y^imCLbFtaU z(w4)P#pZa05Ow3s!RC_z)Qk1J%Pr@3;p;=p9Ts||WhtN^S%~#;hmi}9DHdvK-4)R33P~B3nX_Wh1u^Jz z0d%EVW^M`~&m3(Yia{^Optl2PLF#DJHBIc4dibc>Hh_9l$Cw2QOP%NB zuWcO%=s|%VPTkRZOzH#@@bBTw_gYU#ooLou+V@&dNu6X~51>z^PF6;6J_xyzY1vNX zQqKT7#f(?Tyn_5bZVnRYN2$LE^qfFvn!@l;r9N(^PN&=#^X>wx!n|prxdPQ~Pc7%<_ZR3Oh0K!SUjpaZ0rch6Y36Q$UPv7= z{D9Qy<~e~b>)36(r}L+q-wX7xSbe&ASLGx^PB-}(k^`Uv&FSVafgU#WQos4Ch3JVh z%ufa4i0~b(vc4V2{Rpe93*`B=%B?XkT8MIIn)z6(;CM?9o@EXb=tqV=`?NV*AdbF| zr#@|Nu$;{OGv*x&QOh}|W~OhahcC{Jj6qXl&|HDkmUGRe7NVB(%=Hn@^Uaq7=&98C z=KctDp;>R`Xv@WB)GRFnwOnHMunEW=Z&&zc)7jdGte&qU-tZ{7_!J*>wT zu+a~73%T??g`Y z8f|Z+t}^_BH6YW|_70$f0x0ENZPo-(gL93!BY;LZ*P0ChG|susq<5y4v(2$>Q=RKg zkA+TYo8jDG76|ksL(6V7#|ZSWp_kU0ia;!%hv!#kL^y9U2gx)=Id3)}6-YVPnG*u& ze5~>E)PC+&U=ClFms=T2%;6R@TBbLYo9oqVqTAM-eFp2 z|3CKL1U{-FTl}xOOW#g+IwX)iVNHOrh(HL7q7o98NLUjT6eUf%6IzmVOm|omlwD9j zP(Tn76jV@6c^QRo)~+m>*vOzD)`kVaYyI-X=jty9kEdP9B$jY4F5xg{0%kJ7lvZ>^)DznNq=`BBm;@;(Xg#o|HD|E_#MT1zt2y(CvswE7_igWW3T&{+LQ z&@LJW$KLxSb%|>ZPeAdXN_Np6ao4M#kWoAph?hn_C3891?};<$r)0ax;r_T=)K5ur zZ;Uye+#lCdJwQB+CXvZ;GvRsXK|xF77Lm_MS`H71bKDnXCMU?@E}-R{?0X@HgJip) z&*N@W50WDy-d`ZUgXEl`vQ!R{)|^->Uy@uwpT`ZNUlJcD@)s!Qmt=;Z zQ-bV_?71vIdlaKIP9!mYmU@)*7StsEarGD( zDMGf2e^UL1%nl*jNo!7aoQsc>T;aDh zdJ6cBl(Y_>gU=8l-SOMhZ^;@#_r$*gv`30fBa9ng9~FUFq%Iv{9A{EtAtaw4C_{|c1cpI_0D_}|sv{Lq}#P0 z$tpo533mhS6*MlPNc)ML<7A(lkf&TA(Svvna}vt53#7B4W#MJo&t#aSO6?aiQP6^f z@!CbQQqc22zmna8)Yc++o<;41!MtcjwKBmzE zLDY7KA+~f_4yuJ*N)MmDEy;q~(IJ=S0#OlDcS7bd4bFIZ?Dq zQWvc*JtYWxPF+fd@Z1L^^w#RpR6*Et>d~%}Zr7q|fgtQT(R8dJ>^U)XE+^J=V(BX3 zhdn2jZWn|-CypMJRH?<&bAqtvBvAWMUKZ>*iL{ZRC5hv;B$_P<`%y9-CJ6gceHs+> zV&W970bL{rdsRcaUJ&-GMs&BJt%(n7jp-SappP}7+8sQ1>|-f3S~U%I8BT1*Yf0B|vPUJUv?Z+))HrD-X-W49Y6pI;=qW+n z!LJp)z-U)kUeW@kH4VQL%egD8h!MJRMRs3OxZawkGC#f|yNuD|{Toa{{>uFHL!;sh0Sr)&xbd|-gE!}SMYe#om{Myl@7QgoNl*O+-JSfG| z@ch!LUD6_wPLmn!BA9yzdV};!BN;T`;+H{-8S$&_NXvOh_JpG&ogw3;kdAb-h=(=N ziALXL=AK77(`-h&SS@v-PfNcvas%CL@w0Rl3=Euu7_tXOb*>ffMZUQuS;aS-?}l{bn}J5QO`q9<(z3D1R zxq1%WEh%5Wm7e4Txfknw=&zFedSBYvP2eBjrw*XIC6&{GbjAqNZwwtohZS;@?B+CI>$XeY^vl;QbIf7OS!t=92I#&?ZpoezwVZ2FXUi~+8 zFYS5{rzY0!^a>$}t}9c3rlI=txO1wi$Gy zr24j*bdjW%wuk9@Nu6wu(A|u9pPohcF~aupnKFx>V6=;1?z5@wUd#{udfOhQ@r=0N zT-sR>mh3S)p3zjYtHCNVpPu3b@zTgbdX5pMa+_@-)haMep2`!nBO~-Xt}Ld#Eq+hZ z3DWOpkxRLy`#%=h$8|L+TS6F0gHuG_QV7b*qI|!JD{PF1T8KUj3!Ew!;IzRExfJLuh^z_!vxeA&GDFy6t1C zjpfv)(Kg#YI(8hV%tkwG`{@ZmLmRzk`-EoS$Nk1P`q*}Wo)Pp=qt9($&>{D8zom_i z+PIUQQ^u6gwyzO452N>ZQSvTV{J;UfbisxCEX_pxol3(pVXf`AC%QgN%Q)hV)}&HbQ0KQU4jkD#m+q=}69Xp^d7ig;e0m-4*PRN2e?`0Ap$VqbuM^U0kl z+icC1@e4VPOgU(zDNhLs0JT!WpFqE>g#a@M{Y3ij)JATG>Y`=Oi_>-=l;-i@DDuhap~x(w5Oed;6Fydzmtn5njU<>Q~Bz zA4$!%mn$lv5%k<}gN?xRR%dsf<+~6@;mbQ`R%$sobZ$$B3tL zpYpM!6mp+(f)k8HciZn*;31VP=l#kJf-r~iN)98Q$^>O#H5zG&H$i#R67K8eLi#zlayJEFozT}Sy?3f@M@Lbc>PWDBqW%gOhW=p(TN^mtV3qEn5t*jD+ zsmxJM2*UgDsM7UC49V}qT&08&&wZ|PPEt9Yr=+~Z)5KIBQ*IK3sXVUu81b5)uLKz_ zgy$p^?DLiRg5sJYZImJ5Ijr)wq$&0V%0W){Mok~FKcSpsG>Nomng*YI>9z)QpKiYe zKK}58GDJ|v`tu;UeH!unjIJZA~{l(I?qVeKwe zez*8Nt;Dax{P;aureq1iRF*5l8R30^&tWPhmXOaXPgz2)P&Qjau2iZlA)iw&SVBIp zM6NflF9kk3lq_kreU;LJ5w025+h0^VO1~8NTv2yPtLT@~NcH><5)^1l4c$Gtdu`uGqg+ zE=#h79Z_z1g_o>;v#79R%0Nj;VaFAZq-J5?DKAJ$5BsaKMbgb-XOz8?ZVmfEsk6yU zb4b`vN>fP%VZSIhN-7HbRT(I0RM;hDn4l(5>faQfq$~C-%4kV8_*nQ@k~W4#t0N_C3yW3fOWGZlpsttnNm!D) zPf%L3ufmem%aXneYpABa#`8;S_Cr`>HCNJ;VNKK$NiT#oQ)fxq7}i2vBWYV$EA_CX z-C=Fi@Yl^WKM8BAb{5p8*~PGSYK|o8=%C&qDa_GP^$Y5qI3%o-I!#hR*bV9uNkw5@ z)eVAfY&In9CiOi@1!3LPBa({3x~o44>I-RRsbQOW$@(T1g!NP#N-7HLt!4=74r%77 zeIylx<*IH;MPdEaF@kPw7U}4(&XyGK7^JR{)WDIiZV@yHLJn3xmK5(8qJAr>f#VMK zH$hp6k&Zjn@GZQYS&8wEyVd%V8aN8nHiCvhnr`(LN%0PkIzUnbN0I6lRM;%VQLK)U z)XH&>I$KgZM~S*ZQb$Lrxc8DXO{vxTsFy=j?M;cs&OGEhMuc+D6hj`$9Ea(9_Mv zJDyN)lQiA2L>(z zN!oXk+Bh3%mnGfgOwppYnB!Zl zNjmH3r#&y}7snv&JxNy_gSAtVROjtl*gIx^5zafcmXhL}cWZql6*&vE5=nliN1H8a zw6j=yS<-l?Py1NXG-s)HPSPxAxfZq4%ez}+I*{H zzSdOI7mkHmrlg~eC$+mIopdbK#!1@YSf(wIw8ybR+aPJb<9Y2vNoO6aw9}G)ajenw zcg@_dIM!*6C8^F0+D(!ooUdv_CB->6Yn775JKxaeNt)(-OIs^xmUD;pzN8}OPVHMs ze&>4{-DT!J+PO!oFX=JoUhR5G%bfePOi3>~Kh^RCb%y%MyrO+2Db7jsi;~7W6}{eWGxupun|{5dS|t1pxEm@``6AZeL1Uf(I{MQ4)!nV_4XKI`k}C2evv)?M%OvfKpiGgWUP z=?h0Qy@#Zuj+VMx(n-g4`UFWk9M|iMCGByf>#s`M@93!Smvq+ASwAc37e`k;Y>%1y z6~`@lQ%R~bQ_qwX;q0N`B`MC?TOTKByz^Flfuw29e)mEpry=JtV#8EYxoolneFg)$u2daBR(OzRBU!ACzJtfCH>- zisK>u1xc!NroK~Bgmbq3wWK)bT>UpmYO_IBlUJ*Y( z`%;f&w2+-Fe5uzLgeMD!^_GHgkNy|ElcX#5qk6U^Ti9`ZlBB4xlltS5lEO~wt0XlG zJEMOqDLw3*eo@lRVL$5O`>r-u;1&wN+AFkWpmozNguzfDc8}6{36*Rs1Xh($YiX^|I zjxB6IFZJ~1MUE(2Q$dR%WV9_qlHUC@W?$Qbk|u}ux6P8YFno}0 zg`j2OFNfb|+a&4L@FBJxlHLrz)Ap&LC5bPG54W9=^lG@07CG*-Js`>Nm|%NY(rCwn zwkHJzn-w`G+SW_*JEqvSNgC~#X4@<1IJB4Pwj+}Kj+wU8l14iov0ak%Uid6q3ok69%jVflv!e1ApABqI3K>m_JX9#;md5BBimMzKUdd+r^q*&J*wlR_l9B#o1phDma{j@rs3 z#kx+|?w3^HIBAmG8_w`qrXej6Ly>{5*=Nxk4#!BQplciD}( zl7_j$jir)`U3HB0l3cDRBl$}+%~)5A(Lz#zBi^_{klsAqm1qoBd4vy#Hkw(;4wKLx!=O zhqPacyVsRrY!gJ{kq!#7GrAxsic#cYo*%xipJB8W)Fgh4tD`Z1lf6~^1FlX+iJ(tn zk!A?$$oy6c>dt68CwO0fDug^LX_l+Ap&jA*%}MxccxNM-5x=4?2KwOhj?QVzTwRP* z9uhu}aIfnIqnjYSq8p51g52@X0R;u!6aNy>EJ19I_>X~F3wkmBGoWliJL10r8pa9U`acE~6m%s1l5n7+Q@yB4ly%gAs#_@fN>4W2&SR5qZWUPVo7azeeO6y}x0(+bh%d+XovSPUO+F zOA&*ORg#oCw;As;;bLkUB$*>Nd=C3jI)B4B)VKBhIWEo5m}NL z>nb;*Bo#OUMiW8nB3!OYql2Va*El0nQh{T?(<5kY&P zoSTeiCB?cn8|x$$IJO$w8Sysprm1DFRn97KM zw@8)olpy?Wkt$=8ApHK2Dr2u8{8o@EIvLO7Hjw&PiB<9Y)&!fs{EeOAf zqsqt@gx|eUWegL9-zrgM1Q|`Y<9Aa)euD6ueyR*|id_-;DXnRpDkD|UY**_#?;5$1 zI@H-^j1=@+T8BFC8B+zREjrZMZ7dg57wCOsyPyUwZmhG%I3X#k&R!$@G|#U=i#~Ne zGCBxaS}(88K4XBS4nRRcEnDQ(*>B91)B$J%Bi`daG2Y~Ua93Wb^NF#K(L(!(7F+6k zYW&Lm?3FFv2I}xvmb?A)sCR+N1wGKBHTl$7Bxo+9w>hyiKQ#^rKm6XU1IA@RA413j zM(W>qnpmF)j6R%LyaPso@H^h(NS)7&8G_EXI1co=4EZxq_!*u`WJ{HPX{0ckWKV86 zlYD8k7Jl%vvXNgIS;8;9r41-o_+eQN8->E}=9VcyCBm;y%hu$GF-7?0gWnNjmhii) zr9uBQ-lvVOoZt=b1-8E$3kBgj-Di!}l2(!Ljct;a+RhpK1mQc~ zKN&?oup9`!)BUS4S`fa|eaVsBP*7Rxfsuy&l%Vmgd#ZN3?PnfuJiNplW{;LMB+_A@C1?(W47a~1X(n;m zcME#D^&%2sKf{RkzdH8$zhIgR?Q2^<;HqQaEC|<_b?gTOZEal;S;zjXphJw3FY