Improved support for international text

 - Implementation by @herm for GSOC 2012 (http://mapnik.org/news/2012/10/06/gsoc2012-status9/)
 - C++11 port, improvements, optimizations by @artemp
 - Testing and integration with master by @springmeyer
 - Thank you to all the support from @behdad along the way
 - Thanks for help testing @toton6868, @stephankn, @nirvn, @mfrasca, @simonsonc and many others

Refs: #2073,#2070,#2038,#2037,#1953,#1820,#1819,#1714,#1634,#1547,#1532,#1319,#1208,#1154,#1146

SConstruct

@@ -83,6 +83,7 @@ pretty_dep_names = {
     'png':'PNG C library | configure with PNG_LIBS & PNG_INCLUDES',
     'webp':'WEBP C library | configure with WEBP_LIBS & WEBP_INCLUDES',
     'icuuc':'ICU C++ library | configure with ICU_LIBS & ICU_INCLUDES or use ICU_LIB_NAME to specify custom lib name  | more info: http://site.icu-project.org/',
+    'harfbuzz':'HarfBuzz text shaping library | configure with HB_LIBS & HB_INCLUDES',
     'z':'Z compression library | more info: http://www.zlib.net/',
     'm':'Basic math library, part of C++ stlib',
     'pkg-config':'pkg-config tool | more info: http://pkg-config.freedesktop.org',
@@ -332,7 +333,8 @@ opts.AddVariables(
     PathVariable('ICU_INCLUDES', 'Search path for ICU include files', ICU_INCLUDES_DEFAULT, PathVariable.PathAccept),
     PathVariable('ICU_LIBS','Search path for ICU include files',ICU_LIBS_DEFAULT + LIBDIR_SCHEMA_DEFAULT, PathVariable.PathAccept),
     ('ICU_LIB_NAME', 'The library name for icu (such as icuuc, sicuuc, or icucore)', 'icuuc', PathVariable.PathAccept),
-
+    PathVariable('HB_INCLUDES', 'Search path for HarfBuzz include files', '/usr/include', PathVariable.PathAccept),
+    PathVariable('HB_LIBS','Search path for HarfBuzz include files','/usr/' + LIBDIR_SCHEMA_DEFAULT, PathVariable.PathAccept),
     BoolVariable('PNG', 'Build Mapnik with PNG read and write support', 'True'),
     PathVariable('PNG_INCLUDES', 'Search path for libpng include files', '/usr/include', PathVariable.PathAccept),
     PathVariable('PNG_LIBS','Search path for libpng library files','/usr/' + LIBDIR_SCHEMA_DEFAULT, PathVariable.PathAccept),
@@ -1192,7 +1194,7 @@ if not preconfigured:
 
     # Adding the required prerequisite library directories to the include path for
     # compiling and the library path for linking, respectively.
-    for required in ('ICU', 'SQLITE'):
+    for required in ('ICU', 'SQLITE', 'HB'):
         inc_path = env['%s_INCLUDES' % required]
         lib_path = env['%s_LIBS' % required]
         env.AppendUnique(CPPPATH = os.path.realpath(inc_path))
@@ -1220,6 +1222,7 @@ if not preconfigured:
     REQUIRED_LIBSHEADERS = [
         ['z', 'zlib.h', True,'C'],
         [env['ICU_LIB_NAME'],'unicode/unistr.h',True,'C++'],
+        ['harfbuzz', 'harfbuzz/hb.h',True,'C++']
     ]
 
     OPTIONAL_LIBSHEADERS = []

bindings/python/mapnik/__init__.py

@@ -285,11 +285,6 @@ class _Color(Color,_injector):
     def __repr__(self):
         return "Color(R=%d,G=%d,B=%d,A=%d)" % (self.r,self.g,self.b,self.a)
 
-class _ProcessedText(ProcessedText, _injector):
-    def append(self, properties, text):
-        #More pythonic name
-        self.push_back(properties, text)
-
 class _Symbolizers(Symbolizers,_injector):
 
     def __getitem__(self, idx):

bindings/python/mapnik_symbolizer.cpp

@@ -46,7 +46,7 @@
 #include <mapnik/expression_node.hpp>
 #include <mapnik/value_error.hpp>
 #include <mapnik/marker_cache.hpp> // for known_svg_prefix_
-
+#include <mapnik/pixel_position.hpp>
 
 using mapnik::symbolizer;
 using mapnik::rule;
@@ -66,28 +66,26 @@ using mapnik::path_expression_ptr;
 using mapnik::guess_type;
 using mapnik::expression_ptr;
 using mapnik::parse_path;
-using mapnik::position;
-
 
 namespace {
 using namespace boost::python;
 
 tuple get_shield_displacement(const shield_symbolizer& s)
 {
-    position const& pos = s.get_shield_displacement();
-    return boost::python::make_tuple(pos.first, pos.second);
+    mapnik::pixel_position const& pos = s.get_shield_displacement();
+    return boost::python::make_tuple(pos.x, pos.y);
 }
 
 void set_shield_displacement(shield_symbolizer & s, boost::python::tuple arg)
 {
-    s.get_placement_options()->defaults.displacement.first = extract<double>(arg[0]);
-    s.get_placement_options()->defaults.displacement.second = extract<double>(arg[1]);
+    s.get_placement_options()->defaults.displacement.x = extract<double>(arg[0]);
+    s.get_placement_options()->defaults.displacement.y = extract<double>(arg[1]);
 }
 
 tuple get_text_displacement(const shield_symbolizer& t)
 {
-    position const& pos = t.get_placement_options()->defaults.displacement;
-    return boost::python::make_tuple(pos.first, pos.second);
+    mapnik::pixel_position const& pos = t.get_placement_options()->defaults.displacement;
+    return boost::python::make_tuple(pos.x, pos.y);
 }
 
 void set_text_displacement(shield_symbolizer & t, boost::python::tuple arg)

bindings/python/mapnik_text_placement.cpp

@@ -27,11 +27,13 @@
 #include <boost/noncopyable.hpp>
 
 #include <mapnik/text/text_properties.hpp>
+#include <mapnik/text/placements/simple.hpp>
+#include <mapnik/text/placements/list.hpp>
 #include <mapnik/text/formatting/text.hpp>
 #include <mapnik/text/formatting/list.hpp>
 #include <mapnik/text/formatting/format.hpp>
 #include <mapnik/text/formatting/expression_format.hpp>
-#include <mapnik/text/processed_text.hpp>
+#include <mapnik/text/layout.hpp>
 #include <mapnik/text_symbolizer.hpp>
 
 #include "mapnik_enumeration.hpp"
@@ -96,7 +98,7 @@ class class_with_converter : public boost::python::class_<T, X1, X2, X3>
 
 boost::python::tuple get_displacement(text_symbolizer_properties const& t)
 {
-    return boost::python::make_tuple(t.displacement.first, t.displacement.second);
+    return boost::python::make_tuple(t.displacement.x, t.displacement.y);
 }
 
 void set_displacement(text_symbolizer_properties &t, boost::python::tuple arg)
@@ -112,7 +114,7 @@ void set_displacement(text_symbolizer_properties &t, boost::python::tuple arg)
 
     double x = extract<double>(arg[0]);
     double y = extract<double>(arg[1]);
-    t.displacement = std::make_pair(x, y);
+    t.displacement.set(x, y);
 }
 
 
@@ -123,7 +125,7 @@ struct NodeWrap: formatting::node, wrapper<formatting::node>
 
     }
 
-    void apply(char_properties const& p, feature_impl const& feature, processed_text &output) const
+    void apply(char_properties_ptr p, feature_impl const& feature, text_layout &output) const
     {
         python_block_auto_unblock b;
         this->get_override("apply")(ptr(&p), ptr(&feature), ptr(&output));
@@ -161,7 +163,7 @@ struct TextNodeWrap: formatting::text_node, wrapper<formatting::text_node>
 
     }
 
-    virtual void apply(char_properties const& p, feature_impl const& feature, processed_text &output) const
+    virtual void apply(char_properties_ptr p, feature_impl const& feature, text_layout &output) const
     {
         if(override o = this->get_override("apply"))
         {
@@ -174,15 +176,15 @@ struct TextNodeWrap: formatting::text_node, wrapper<formatting::text_node>
         }
     }
 
-    void default_apply(char_properties const& p, feature_impl const& feature, processed_text &output) const
+    void default_apply(char_properties_ptr p, feature_impl const& feature, text_layout &output) const
     {
         formatting::text_node::apply(p, feature, output);
     }
 };
 
 struct FormatNodeWrap: formatting::format_node, wrapper<formatting::format_node>
 {
-    virtual void apply(char_properties const& p, feature_impl const& feature, processed_text &output) const
+    virtual void apply(char_properties_ptr p, feature_impl const& feature, text_layout &output) const
     {
         if(override o = this->get_override("apply"))
         {
@@ -195,15 +197,15 @@ struct FormatNodeWrap: formatting::format_node, wrapper<formatting::format_node>
         }
     }
 
-    void default_apply(char_properties const& p, feature_impl const& feature, processed_text &output) const
+    void default_apply(char_properties_ptr p, feature_impl const& feature, text_layout &output) const
     {
         formatting::format_node::apply(p, feature, output);
     }
 };
 
 struct ExprFormatWrap: formatting::expression_format, wrapper<formatting::expression_format>
 {
-    virtual void apply(char_properties const& p, feature_impl const& feature, processed_text &output) const
+    virtual void apply(char_properties_ptr p, feature_impl const& feature, text_layout &output) const
     {
         if(override o = this->get_override("apply"))
         {
@@ -216,7 +218,7 @@ struct ExprFormatWrap: formatting::expression_format, wrapper<formatting::expres
         }
     }
 
-    void default_apply(char_properties const& p, feature_impl const& feature, processed_text &output) const
+    void default_apply(char_properties_ptr p, feature_impl const& feature, text_layout &output) const
     {
         formatting::expression_format::apply(p, feature, output);
     }
@@ -243,7 +245,7 @@ struct ListNodeWrap: formatting::list_node, wrapper<formatting::list_node>
     /* TODO: Add constructor taking variable number of arguments.
        http://wiki.python.org/moin/boost.python/HowTo#A.22Raw.22_function */
 
-    virtual void apply(char_properties const& p, feature_impl const& feature, processed_text &output) const
+    virtual void apply(char_properties_ptr p, feature_impl const& feature, text_layout &output) const
     {
         if(override o = this->get_override("apply"))
         {
@@ -256,7 +258,7 @@ struct ListNodeWrap: formatting::list_node, wrapper<formatting::list_node>
         }
     }
 
-    void default_apply(char_properties const& p, feature_impl const& feature, processed_text &output) const
+    void default_apply(char_properties_ptr p, feature_impl const& feature, text_layout &output) const
     {
         formatting::list_node::apply(p, feature, output);
     }
@@ -322,12 +324,12 @@ void insert_expression(expression_set *set, expression_ptr p)
     set->insert(p);
 }
 
-char_properties & get_format(text_symbolizer const& sym)
+char_properties_ptr get_format(text_symbolizer const& sym)
 {
     return sym.get_placement_options()->defaults.format;
 }
 
-void set_format(text_symbolizer const& sym, char_properties & format)
+void set_format(text_symbolizer const& sym, char_properties_ptr format)
 {
     sym.get_placement_options()->defaults.format = format;
 }
@@ -395,7 +397,7 @@ void export_text_placement()
                       &text_symbolizer::set_placement_options)
         //TODO: Check return policy, is there a better way to do this?
         .add_property("format",
-                      make_function(&get_format, return_value_policy<reference_existing_object>()),
+                      make_function(&get_format),
                       &set_format,
                       "Shortcut for placements.defaults.default_format")
         .add_property("properties",
@@ -439,6 +441,7 @@ void export_text_placement()
         .def_readwrite("largest_bbox_only", &text_symbolizer_properties::largest_bbox_only)
         .def_readwrite("text_ratio", &text_symbolizer_properties::text_ratio)
         .def_readwrite("wrap_width", &text_symbolizer_properties::wrap_width)
+        .def_readwrite("wrap_before", &text_symbolizer_properties::wrap_before)
         .def_readwrite("format", &text_symbolizer_properties::format)
         .add_property ("format_tree",
                        &text_symbolizer_properties::format_tree,
@@ -451,7 +454,8 @@ void export_text_placement()
     ;
 
 
-    class_with_converter<char_properties>
+    class_with_converter<char_properties,
+            std::shared_ptr<char_properties> >
         ("CharProperties")
         .def_readwrite_convert("text_transform", &char_properties::text_transform)
         .def_readwrite_convert("fontset", &char_properties::fontset)
@@ -463,7 +467,6 @@ void export_text_placement()
         .def_readwrite("text_opacity", &char_properties::text_opacity)
         .def_readwrite("wrap_char", &char_properties::wrap_char)
         .def_readwrite("wrap_character", &char_properties::wrap_char)
-        .def_readwrite("wrap_before", &char_properties::wrap_before)
         .def_readwrite("fill", &char_properties::fill)
         .def_readwrite("halo_fill", &char_properties::halo_fill)
         .def_readwrite("halo_radius", &char_properties::halo_radius)
@@ -486,24 +489,12 @@ void export_text_placement()
         ("TextPlacementInfo",
          init<text_placements const*, double>())
         .def("next", pure_virtual(&text_placement_info::next))
-        .def("get_actual_label_spacing", &text_placement_info::get_actual_label_spacing)
-        .def("get_actual_minimum_distance", &text_placement_info::get_actual_minimum_distance)
-        .def("get_actual_minimum_padding", &text_placement_info::get_actual_minimum_padding)
         .def_readwrite("properties", &text_placement_info::properties)
         .def_readwrite("scale_factor", &text_placement_info::scale_factor)
         ;
     register_ptr_to_python<std::shared_ptr<text_placement_info> >();
 
 
-    class_<processed_text,
-        std::shared_ptr<processed_text>,
-        boost::noncopyable>
-        ("ProcessedText", no_init)
-        .def("push_back", &processed_text::push_back)
-        .def("clear", &processed_text::clear)
-        ;
-
-
     class_<expression_set,
         std::shared_ptr<expression_set>,
         boost::noncopyable>
@@ -551,7 +542,6 @@ void export_text_placement()
         .def_readwrite_convert("text_opacity", &formatting::format_node::text_opacity)
         .def_readwrite_convert("wrap_char", &formatting::format_node::wrap_char)
         .def_readwrite_convert("wrap_character", &formatting::format_node::wrap_char)
-        .def_readwrite_convert("wrap_before", &formatting::format_node::wrap_before)
         .def_readwrite_convert("text_transform", &formatting::format_node::text_transform)
         .def_readwrite_convert("fill", &formatting::format_node::fill)
         .def_readwrite_convert("halo_fill", &formatting::format_node::halo_fill)
@@ -591,7 +581,6 @@ void export_text_placement()
         .def_readwrite("text_opacity", &formatting::expression_format::text_opacity)
         .def_readwrite("wrap_char", &formatting::expression_format::wrap_char)
         .def_readwrite("wrap_character", &formatting::expression_format::wrap_char)
-        .def_readwrite("wrap_before", &formatting::expression_format::wrap_before)
         .def_readwrite("fill", &formatting::expression_format::fill)
         .def_readwrite("halo_fill", &formatting::expression_format::halo_fill)
         .def_readwrite("halo_radius", &formatting::expression_format::halo_radius)

include/mapnik/box2d.hpp

@@ -97,12 +97,15 @@ template <typename T> class MAPNIK_DECL box2d
     void pad(T padding);
     bool from_string(std::string const& str);
     bool valid() const;
+    void move(T x, T y);
 
     // define some operators
     box2d_type& operator+=(box2d_type const& other);
     box2d_type& operator*=(T);
     box2d_type& operator/=(T);
     T operator[](int index) const;
+    box2d_type operator +(T other) const; //enlarge box by given amount
+    box2d_type& operator +=(T other); //enlarge box by given amount
 
     // compute the bounding box of this one transformed
     box2d_type  operator* (agg::trans_affine const& tr) const;

include/mapnik/cairo_context.hpp

@@ -33,6 +33,9 @@
 #include <mapnik/image_compositing.hpp>
 #include <mapnik/font_engine_freetype.hpp>
 #include <mapnik/gradient.hpp>
+#include <mapnik/text/text_properties.hpp>
+#include <mapnik/text/placements_list.hpp>
+#include <mapnik/text/glyph_info.hpp>
 #include <mapnik/vertex.hpp>
 #include <mapnik/noncopyable.hpp>
 
@@ -317,9 +320,9 @@ class cairo_context : private mapnik::noncopyable
     void translate(double x, double y);
     void save();
     void restore();
-    void show_glyph(unsigned long index, double x, double y);
-    void glyph_path(unsigned long index, double x, double y);
-    void add_text(text_path const& path,
+    void show_glyph(unsigned long index, pixel_position const& pos);
+    void glyph_path(unsigned long index, pixel_position const& pos);
+    void add_text(glyph_positions_ptr pos,
                   cairo_face_manager & manager,
                   face_manager<freetype_engine> & font_manager,
                   double scale_factor = 1.0);