Merge branch 'master' into 2340-privacy-dialog-loading-err-states

lib/assets/javascripts/cartodb/new_dashboard/dialogs/mamufas_import_dialog_view.js

@@ -23,6 +23,17 @@ module.exports = BaseDialog.extend({
     this.elder('render');
     this.$('.Dialog-content').addClass('Dialog-content--expanded');
     return this;
+  },
+
+  show: function() {
+    // Don't send any kind of openDialog signal
+    this.$el.show();
+    this.trigger('show');
+
+    // Blur current element (e.g. a <a> tag that was clicked to open this window)
+    if (document.activeElement) {
+      document.activeElement.blur();
+    }
   }
 
 });

services/importer/lib/importer/shp_normalizer.rb

@@ -7,6 +7,53 @@ module CartoDB
   module Importer2
     class ShpNormalizer
       DEFAULT_ENCODING = 'LATIN1'
+
+      # INFO: http://www.postgresql.org/docs/9.1/static/multibyte.html
+      SUPPORTED_ENCODINGS = %W{
+        BIG5 WIN950 Windows950
+        EUC_CN
+        EUC_JP
+        EUC_JIS_2004
+        EUC_KR
+        EUC_TW
+        GB18030
+        GBK WIN936 Windows936
+        ISO_8859_5
+        ISO_8859_6 ISO_8859_7
+        ISO_8859_8
+        JOHAB
+        KOI8R KOI8
+        KOI8U
+        LATIN1 ISO88591 ISO-8859-1
+        LATIN2 ISO88592
+        LATIN3 ISO88593
+        LATIN4 ISO88594
+        LATIN5 ISO88599
+        LATIN6 ISO885910
+        LATIN7 ISO885913
+        LATIN8 ISO885914
+        LATIN9 ISO885915
+        LATIN10 ISO885916
+        MULE_INTERNAL
+        SJIS Mskanji ShiftJIS WIN932 Windows932
+        SHIFT_JIS_2004
+        SQL_ASCII
+        UHC WIN949 Windows949
+        UTF8 Unicode UTF-8
+        WIN866 ALT
+        WIN874
+        WIN1250
+        WIN1251 WIN
+        WIN1252
+        WIN1253
+        WIN1254
+        WIN1255
+        WIN1256
+        WIN1257
+        WIN1258 ABC TCVN TCVN5712 VSCII
+      }
+      SUPPORTED_ENCODINGS_DOWNCASED = SUPPORTED_ENCODINGS.map(&:downcase)
+
       NORMALIZER_RELATIVE_PATH = 
         "../../../../../lib/importer/misc/shp_normalizer.py"
 
@@ -31,14 +78,38 @@ def shape_encoding
       def shape_encoding_guessing
         normalize
         dbf       = filepath.gsub(%r{\.shp$}, '.dbf')
-        encoding  = DBF::Table.new(dbf).encoding || 
+        encoding  = read_encoding_files || DBF::Table.new(dbf).encoding ||
                     normalizer_output.fetch(:encoding, nil)
         encoding  = DEFAULT_ENCODING if encoding == 'None' 
         encoding  = codepage_for(encoding) if windows?(encoding)
         return(tab_encoding || encoding) if tab?
         encoding
       end
 
+      # http://gis.stackexchange.com/questions/3529/which-character-encoding-is-used-by-the-dbf-file-in-shapefiles
+      # ArcGIS and Geopublisher, AtlasStyler and Geoserver: .cpg
+      # Geoserver: cst
+      def read_encoding_files
+        filter_supported_encodings(read_encoding_file('cpg') || read_encoding_file('cst'))
+      end
+
+      def filter_supported_encodings(encoding)
+        encoding.nil? || !SUPPORTED_ENCODINGS_DOWNCASED.include?(encoding.downcase) ? nil : encoding
+      end
+
+      def read_encoding_file(extension)
+        current_extension = File.extname(filepath)
+        path = filepath.gsub(/#{current_extension}$/, ".#{extension}")
+        return nil unless File.exists?(path)
+        saved_encoding = nil
+        f = File.open(path, 'r') { |file|
+          saved_encoding = file.read
+        }
+        saved_encoding
+      rescue => e
+        nil
+      end
+
       def tab_encoding
         return 'WIN1251' if File.open(filepath, 'rb') { |file|
           file.read =~ /WindowsCyrillic/

services/importer/spec/fixtures/greek/greek.cpg

@@ -0,0 +1 @@
+greek

services/importer/spec/fixtures/greek/greek.prj

@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]

services/importer/spec/fixtures/greek/greek.qpj

@@ -0,0 +1 @@
+GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]

services/importer/spec/unit/shp_normalizer_spec.rb

@@ -11,15 +11,24 @@
 
   describe '#shape_encoding' do
 
-    it 'guesses ISO-8859-1 encoding for USA counties common data unzipped' do
+    it 'guesses UTF-8 encoding for USA counties common data unzipped with cpg file' do
       job = CartoDB::Importer2::Doubles::Job.new
       job.stubs(:table_name).returns('county_usa')
       path = File.expand_path(File.join(File.dirname(__FILE__), "../fixtures/county_usa/county_usa.shp"))
       shp_normalizer = CartoDB::Importer2::ShpNormalizer.new(path, job)
 
-      shp_normalizer.shape_encoding.should eq 'WIN1252'
+      shp_normalizer.shape_encoding.should eq 'UTF-8'
     end
 
+    it 'guesses LATIN1 encoding for a "greek", unsupported encoding' do
+      job = CartoDB::Importer2::Doubles::Job.new
+      job.stubs(:table_name).returns('greek')
+      path = File.expand_path(File.join(File.dirname(__FILE__), "../fixtures/greek/greek.shp"))
+      shp_normalizer = CartoDB::Importer2::ShpNormalizer.new(path, job)
+
+      shp_normalizer.shape_encoding.should eq 'LATIN1'
+    end
+
   end
 
 end

spec/models/visualization/member_spec.rb

@@ -1024,6 +1024,8 @@
     end
 
     it "Returns the vizjson if it was cached before" do
+      pending "Vizjson caching disabled, see https://github.com/CartoDB/cartodb/issues/2407#issuecomment-75820947"
+
       member = Visualization::Member.new(random_attributes_for_vis_member(user_id: @user_mock.id))
       member.store
       mocked_vizjson = {mocked: 'vizjson'}
@@ -1122,6 +1124,8 @@
 
   describe '#invalidate_redis_cache' do
     it "Invalidates the vizjson in redis cache" do
+      pending "vizjson caching disabled, see https://github.com/CartoDB/cartodb/issues/2407#issuecomment-75820947"
+
       member = Visualization::Member.new(random_attributes_for_vis_member(user_id: @user_mock.id))
       member.store
       mocked_vizjson = {mocked: 'vizjson'}