Python Unicode objects are now handled

src/py_utils.cc

@@ -61,7 +61,8 @@ struct bool_from_python
   static void construct(PyObject* obj_ptr,
 			converter::rvalue_from_python_stage1_data* data)
   {
-    void* storage = ((converter::rvalue_from_python_storage<bool>*) data)->storage.bytes;
+    void * storage =
+      ((converter::rvalue_from_python_storage<bool>*) data)->storage.bytes;
     if (obj_ptr == Py_True)
       new (storage) bool(true);
     else
@@ -92,23 +93,72 @@ struct string_from_python
     return obj_ptr;
   }
 
-  static void construct(PyObject* obj_ptr, converter::rvalue_from_python_stage1_data* data)
+  static void construct(PyObject* obj_ptr,
+			converter::rvalue_from_python_stage1_data* data)
   {
     const char* value = PyString_AsString(obj_ptr);
     if (value == 0) throw_error_already_set();
     void* storage =
-      reinterpret_cast<converter::rvalue_from_python_storage<ledger::string> *>(data)->storage.bytes;
+      reinterpret_cast<converter::rvalue_from_python_storage<ledger::string> *>
+                      (data)->storage.bytes;
     new (storage) ledger::string(value);
     data->convertible = storage;
   }
 };
 
-typedef register_python_conversion<ledger::string, string_to_python, string_from_python>
+typedef register_python_conversion<ledger::string,
+				   string_to_python, string_from_python>
   string_python_conversion;
 
 #endif // STRING_VERIFY_ON
 
 
+struct unicode_to_python
+{
+  static PyObject* convert(const std::string& utf8str)
+  {
+    PyObject * pstr = PyString_FromString(utf8str.c_str());
+    PyObject * uni  = PyUnicode_FromEncodedObject(pstr, "UTF-8", NULL);
+    return object(handle<>(borrowed(uni))).ptr();
+  }
+};
+
+struct unicode_from_python
+{
+  static void* convertible(PyObject* obj_ptr)
+  {
+    if (!PyUnicode_Check(obj_ptr)) return 0;
+    return obj_ptr;
+  }
+
+  static void construct(PyObject* obj_ptr,
+			converter::rvalue_from_python_stage1_data* data)
+  {
+    Py_ssize_t size = PyUnicode_GET_SIZE(obj_ptr);
+    const Py_UNICODE* value = PyUnicode_AS_UNICODE(obj_ptr);
+
+    std::string str;
+    if (sizeof(Py_UNICODE) == 2) // UTF-16
+      utf8::unchecked::utf16to8(value, value + size, std::back_inserter(str));
+    else if (sizeof(Py_UNICODE) == 4) // UTF-32
+      utf8::unchecked::utf32to8(value, value + size, std::back_inserter(str));
+    else
+      assert(! "Py_UNICODE has an unexpected size");
+
+    if (value == 0) throw_error_already_set();
+    void* storage =
+      reinterpret_cast<converter::rvalue_from_python_storage<std::string> *>
+                      (data)->storage.bytes;
+    new (storage) std::string(str);
+    data->convertible = storage;
+  }
+};
+
+typedef register_python_conversion<std::string,
+				   unicode_to_python, unicode_from_python>
+  unicode_python_conversion;
+
+
 struct istream_to_python
 {
   static PyObject* convert(const std::istream&)
@@ -125,16 +175,19 @@ struct istream_from_python
     return obj_ptr;
   }
 
-  static void construct(PyObject* obj_ptr, converter::rvalue_from_python_stage1_data* data)
+  static void construct(PyObject* obj_ptr,
+			converter::rvalue_from_python_stage1_data* data)
   {
     void* storage =
-      reinterpret_cast<converter::rvalue_from_python_storage<pyifstream> *>(data)->storage.bytes;
+      reinterpret_cast<converter::rvalue_from_python_storage<pyifstream> *>
+                      (data)->storage.bytes;
     new (storage) pyifstream(reinterpret_cast<PyFileObject *>(obj_ptr));
     data->convertible = storage;
   }
 };
 
-typedef register_python_conversion<std::istream, istream_to_python, istream_from_python>
+typedef register_python_conversion<std::istream,
+				   istream_to_python, istream_from_python>
   istream_python_conversion;
 
 
@@ -154,15 +207,19 @@ struct ostream_from_python
     return obj_ptr;
   }
 
-  static void construct(PyObject* obj_ptr, converter::rvalue_from_python_stage1_data* data)
+  static void construct(PyObject* obj_ptr,
+			converter::rvalue_from_python_stage1_data* data)
   {
-    void* storage = reinterpret_cast<converter::rvalue_from_python_storage<pyofstream> *>(data)->storage.bytes;
+    void* storage =
+      reinterpret_cast<converter::rvalue_from_python_storage<pyofstream> *>
+                      (data)->storage.bytes;
     new (storage) pyofstream(reinterpret_cast<PyFileObject *>(obj_ptr));
     data->convertible = storage;
   }
 };
 
-typedef register_python_conversion<std::ostream, ostream_to_python, ostream_from_python>
+typedef register_python_conversion<std::ostream,
+				   ostream_to_python, ostream_from_python>
   ostream_python_conversion;
 
 
@@ -219,6 +276,7 @@ void export_utils()
 #if defined(STRING_VERIFY_ON)
   string_python_conversion();
 #endif
+  unicode_python_conversion();
   istream_python_conversion();
   ostream_python_conversion();
 }

src/system.hh.in

@@ -246,6 +246,7 @@ void serialize(Archive& ar, istream_pos_type& pos, const unsigned int)
 
 #include <boost/python/detail/wrap_python.hpp>
 #include <datetime.h>
+#include <unicodeobject.h>
 
 #include <boost/python/module_init.hpp>