From 9f3ff68244df62e045a96900980637a4825a0cc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charle=CC=81ric=20Paccard?= Date: Tue, 1 May 2012 21:37:38 +0200 Subject: [PATCH] New importer pages immo WS et et pages immo FTP --- lib/importers.rb | 2 + lib/importers/pagesimmo_ftp.rb | 204 +++++++++++++++++++++++++++++++++ lib/importers/pagesimmo_ws.rb | 174 ++++++++++++++++++++++++++++ 3 files changed, 380 insertions(+) create mode 100644 lib/importers/pagesimmo_ftp.rb create mode 100644 lib/importers/pagesimmo_ws.rb diff --git a/lib/importers.rb b/lib/importers.rb index 593dfb4..3140573 100644 --- a/lib/importers.rb +++ b/lib/importers.rb @@ -11,6 +11,8 @@ module Importers 'ics' => Importers::Ics, 'nexto' => Importers::Nexto, 'immoselect' => Importers::Immoselect, + 'pagesimmows' => Importers::PagesimmoWs, + 'pagesimmoftp' => Importers::PagesimmoFtp, 'enova' => Importers::Enova, # 'sitra' => Importers::Sitra, # 'rodacom' => Importers::Rodacom, diff --git a/lib/importers/pagesimmo_ftp.rb b/lib/importers/pagesimmo_ftp.rb new file mode 100644 index 0000000..dcadd8c --- /dev/null +++ b/lib/importers/pagesimmo_ftp.rb @@ -0,0 +1,204 @@ +class Importers::PagesimmoFtp < Importers::FromFiles + + #waiting parameters : code_agence + def initialize passerelle + super passerelle, %w(application/zip) + end + + TOTAL_FTP_FOLDER = "pagesimmo/" + + def scan_files + Logger.send("warn","[PagesimmoFtp] Start scan files") + @result[:description] << "[PagesimmoFtp] Start scan files" + + unless File.exists? $base_ftp_repo+TOTAL_FTP_FOLDER + Logger.send("warn","[PagesimmoFtp] Directory #{TOTAL_FTP_FOLDER} does not exist.") + @result[:description] << "[PagesimmoFtp] Directory #{TOTAL_FTP_FOLDER} does not exist." + return + end + + # obtain a list of recent zips, sorted by modification time, + # that can be opened as zipfiles, and save them + pattern = File.join($base_ftp_repo+TOTAL_FTP_FOLDER, '*') + Dir[pattern].sort { |a,b| + File.mtime(a) <=> File.mtime(b) + }.select{ |path| + tmp_file = File.new(path,"r+b") + filename = File.basename path + nb_good_file = 0 + nb_file_read = 0 + # rescue if error during reading zipfile (ex : zip not found) + begin + z = Zip::ZipFile.open(path) + z.each do |entry| + nb_good_file += 1 if entry.name.to_s.downcase =~ /(#{@parameters["code_agence"].to_s}).*\.(jpg|jpeg|png|bmp)$/ + nb_file_read += 1 + break if nb_file_read >= 10 + end + nb_good_file > 0 + rescue + @logger.warn "Error during reading zip file : #{path}" + false + end + ExecutionSourceFile.where(:hashsum => (Digest::MD5.hexdigest tmp_file.read)).select{ |e| e.execution && e.execution.passerelle == @passerelle }.empty? + }.each { |path| + # self.from_file(filename,file,execution) + name = path + tmp_file = File.new(path,"r+b") + e = Execution.new + e.passerelle = @passerelle + e.statut = "nex" + e.save! + f = ExecutionSourceFile.from_file(name,tmp_file,e) + e.execution_source_file = f + e.save! + } + end + # Import/update good information + def import_exe execution + Logger.send("warn","[PagesimmoFtp] Starting PagesimmoWs execution import") + @result[:description] << "[PagesimmoFtp] Starting PagesimmoWs execution import" + + z = Zip::ZipFile.open(execution.execution_source_file.file.path) + + dl_and_update_medias z + + z.each do |entry| + next unless entry.name.downcase =~ /.cryptml$/ + data = z.read(entry) + tree = Hash.from_xml(data) + import_hash(tree) + end + + z.close + + maj_biens + + Logger.send("warn","[PagesimmoFtp] Finished PagesimmoWs execution import") + @result[:description] << "[PagesimmoFtp] Finished PagesimmoWs execution import" + return @result + end + + def import_hash hashtree + Logger.send("warn","[PagesimmoFtp] Start XML(hash) import") + @result[:description] << "[PagesimmoFtp] Start XML(hash) import" + + goods = hashtree['ROOT']['DESTINATAIRE']['AGENCE']['BIEN'] + return if goods.nil? + goods = [goods] unless goods.kind_of? Array + # Create and list new goods + goods.each { |b| + import_bien b + } + + Logger.send("warn","[PagesimmoFtp] End XML import") + @result[:description] << "[PagesimmoFtp] End XML import" + return true + end + + def find_cat(b) + ["APPARTEMENT","MAISON","DEMEURE_CHATEAU","TERRAIN","PARKING_GARAGE","IMMEUBLE","LOCAL_COMMERCIAL"].each { |cat| + return cat unless b[cat].nil? + } + return nil + end + + # Create a new good, and return the ActiveRecord + def import_bien b + + # Good location + good_address = {} + loc = BienEmplacement.new + loc.pays = "France" + loc.code_postal = b['LOCALISATION']['CP'] + loc.ville = b['LOCALISATION']['VILLE'] + + # Category & Transaction type + cat_text = find_cat(b) + if(cat_text.nil? or b[cat_text].nil?) + Logger.send("warn","[PagesimmoWs] Categorie non connue pour le bien ref : #{b["REFERENCE"]}") + @result[:description] << "[PagesimmoWs] Categorie non connue pour le bien ref : #{b["REFERENCE"]}" + return + end + cat_root = b[cat_text] + cat = BienType.find_or_create cat_text.up_first + + if !(b["VENTE"].nil?) + transaction_type = BienTransaction.where(:nom => 'Vente').first + transactionTypeIndex = "VENTE" + price = b["VENTE"]["PRIX"] + elsif !(b["LOCATION"].nil?) + transaction_type = BienTransaction.where(:nom => 'Location').first + transactionTypeIndex = "LOCATION" + b["LOCATION"]["PROVISION_SUR_CHARGES"] ||= 0 + price = b["LOCATION"]["LOYER"].to_i + b["LOCATION"]["PROVISION_SUR_CHARGES"].to_i + # elsif !(b["SAISONNIER"].nil?) + # transaction_type = Immo::TransactionType.get 'Saisonnier' + # transactionTypeIndex = "SAISONNIER" + # price = 0 + else + Logger.send("warn","[PagesimmoWs] Type de transaction nul pour le bien ref : #{b["REFERENCE"]}") + @result[:description] << "[PagesimmoWs] Type de transaction nul pour le bien ref : #{b["REFERENCE"]}" + return + end + + # find if good already exist, unless create it + ref = b["REFERENCE"] + nb = Bien.where(:reference => ref).select{ |b| b.passerelle.installation == @passerelle.installation }.first + nb = Bien.new if nb.nil? + + nb.is_accueil = false + #nb.is_accueil = true if b["TEXTE_MAILING"] && (b["TEXTE_MAILING"].to_s.downcase =~ /.*virtual.*touch.*/) + + # update attributes + nb.passerelle = @passerelle + nb.reference = ref + nb.bien_type = cat + nb.bien_transaction = transaction_type + nb.bien_emplacement = loc + + nb.nb_piece = cat_root['NB_PIECES'].to_i + nb.nb_chambre = cat_root['NB_CHAMBRES'].to_i + nb.surface = cat_root['SURFACE_HABITABLE'].to_i + nb.titre = cat_text + + nb.prix = price + nb.description = b[transactionTypeIndex]["TEXTES"]["DESCRIPTION_FR"] + if b['DIAGNOSTICS'] && b['DIAGNOSTICS']['DPE_CONSOMMATIONS_ENERGIE'] && b['DIAGNOSTICS']['DPE_CONSOMMATIONS_ENERGIE']['DONNEES'] + nb.valeur_dpe = b['DIAGNOSTICS']['DPE_CONSOMMATIONS_ENERGIE']['DONNEES']['LETTRE'] + nb.classe_dpe = b['DIAGNOSTICS']['DPE_CONSOMMATIONS_ENERGIE']['DONNEES']['VALEUR'] + end + if b['DIAGNOSTICS'] && b['DIAGNOSTICS']['DPE_EMISSIONS_GES'] && b['DIAGNOSTICS']['DPE_CONSOMMATIONS_ENERGIE']['DONNEES'] + nb.valeur_ges = b['DIAGNOSTICS']['DPE_EMISSIONS_GES']['DONNEES']['VALEUR'] + nb.class_ges = b['DIAGNOSTICS']['DPE_EMISSIONS_GES']['DONNEES']['VALEUR'] + end + nb.statut = 'new' + nb.save! + + # If new images : Drop old images, add current images + if b['FICHIER'] + pl = b['FICHIER'] + pl = [pl] unless pl.kind_of? Array + if b['FICHIER'].first['FICHIER_JOINT'] + list_hash = pl.select{ |pf| pf['INDICE'].to_i != 0 && pf['FICHIER_JOINT'] && pf['FICHIER_JOINT']['NOM_FICHIER'] && !pf['FICHIER_JOINT']['NOM_FICHIER'].empty? } + if !list_hash.empty? + list_img = list_hash.map{ |ih| ih['FICHIER_JOINT']['NOM_FICHIER'] } + # un-attach old + nb.bien_photos.each{ |photo| + photo.bien = nil + photo.save! + } + # Map photo-hashes to medias, filter out failures, and add medias to good + counter = 0 + list_img.map{ |p| import_local_media(p.to_s,(counter+=1),nb,p.to_s) } + end + end + end + nb.save! + + return true + + return + end + +end \ No newline at end of file diff --git a/lib/importers/pagesimmo_ws.rb b/lib/importers/pagesimmo_ws.rb new file mode 100644 index 0000000..1167175 --- /dev/null +++ b/lib/importers/pagesimmo_ws.rb @@ -0,0 +1,174 @@ +class Importers::PagesimmoWs < Importers::FromUrls + + URI_GOODS_PI_WS = "http://v2.gercop-transac.net/office2/%s/cache/export.xml" + + #waiting parameters : code_agence + + def create_uri + return URI_GOODS_PI_WS % [@parameters["code_agence"].to_s] + end + + # Re-write scan files for PagesimmoWs importer + # Delete the line with the date of call web service + def scan_files + Logger.send("warn","[PagesimmoWs] Start scan files") + @result[:description] << "[PagesimmoWs] Start scan files" + + uri = create_uri + Logger.send("warn","Loading from URI #{uri}") + @result[:description] << "Loading from URI #{uri}" + data = "" + begin + uriOpener = open(URI.encode(uri),"r:utf-8",{:read_timeout => 1800}) + data = uriOpener.read + ### check if the file is a well formated xml ### + rescue + Logger.send("warn","Failure: (#{$!})") + @result[:description] << "Failure: (#{$!})" + return false + end + #data.gsub!(/^.*.*<\/DATE_CREATION>.*$/,"") + # check if the xlm returned by the uri have been already downloaded + + return unless ExecutionSourceFile.where(:hashsum => (Digest::MD5.hexdigest data)).select{ |e| e.execution && e.execution.passerelle == @passerelle }.empty? + e = Execution.new + e.passerelle = @passerelle + e.statut = "nex" + e.save! + + f = ExecutionSourceFile.from_data("data_file", data, e) + + e.execution_source_file = f + e.save! + end + + # Import/update good information + def import_exe execution + Logger.send("warn","[PagesimmoWs] Starting PagesimmoWs execution import") + @result[:description] << "[PagesimmoWs] Starting PagesimmoWs execution import" + + f = File.open(execution.execution_source_file.file.path) + data = f.read + hash = Hash.from_xml(data) + + # Create and list new goods + goods = hash['LISTEPA']['BIEN'] + return if goods.nil? + goods = [goods] unless goods.kind_of? Array + + update_remote_medias + + Logger.send("warn","[PagesimmoWs] #{goods.size} goods in XML stream.") + @result[:description] << "[PagesimmoWs] #{goods.size} goods in XML stream." + goods.each { |b| import_bien b } + + maj_biens + + Logger.send("warn","[PagesimmoWs] Finished PagesimmoWs execution import") + @result[:description] << "[PagesimmoWs] Finished PagesimmoWs execution import" + return @result + end + + def find_cat(b) + ["APPARTEMENT","MAISON","DEMEURE_CHATEAU","TERRAIN","PARKING_GARAGE","IMMEUBLE","LOCAL_COMMERCIAL"].each { |cat| + return cat unless b[cat].nil? + } + return nil + end + + # Create a new good, and return the ActiveRecord + def import_bien b + + # Good location + good_address = {} + if b['LOCALISATION']['VISIBLE'].to_s == "true" + loc = BienEmplacement.new + loc.pays = "France" + loc.code_postal = b['LOCALISATION']['CODE_POSTAL'] + loc.ville = b['LOCALISATION']['VILLE'] + else + loc = nil + end + + # Category & Transaction type + cat_text = find_cat(b) + if(cat_text.nil? or b[cat_text].nil?) + Logger.send("warn","[PagesimmoWs] Categorie non connue pour le bien ref : #{b["INFO_GENERALES"]["AFF_NUM"]}") + @result[:description] << "[PagesimmoWs] Categorie non connue pour le bien ref : #{b["INFO_GENERALES"]["AFF_NUM"]}" + return + end + cat_root = b[cat_text] + cat = BienType.find_or_create cat_text.up_first + + if !(b["VENTE"].nil?) + transaction_type = BienTransaction.where(:nom => 'Vente').first + transactionTypeIndex = "VENTE" + price = b["VENTE"]["PRIX"] + elsif !(b["LOCATION"].nil?) + transaction_type = BienTransaction.where(:nom => 'Location').first + transactionTypeIndex = "LOCATION" + b["LOCATION"]["PROVISION_SUR_CHARGES"] ||= 0 + price = b["LOCATION"]["LOYER"].to_i + b["LOCATION"]["PROVISION_SUR_CHARGES"].to_i + # elsif !(b["SAISONNIER"].nil?) + # transaction_type = Immo::TransactionType.get 'Saisonnier' + # transactionTypeIndex = "SAISONNIER" + # price = 0 + else + Logger.send("warn","[PagesimmoWs] Type de transaction nul pour le bien ref : #{b["INFO_GENERALES"]["AFF_NUM"]}") + @result[:description] << "[PagesimmoWs] Type de transaction nul pour le bien ref : #{b["INFO_GENERALES"]["AFF_NUM"]}" + return + end + + # find if good already exist, unless create it + ref = b["INFO_GENERALES"]["AFF_NUM"] + nb = Bien.where(:reference => ref).select{ |b| b.passerelle.installation == @passerelle.installation }.first + nb = Bien.new if nb.nil? + + nb.is_accueil = false + #nb.is_accueil = true if b["TEXTE_MAILING"] && (b["TEXTE_MAILING"].to_s.downcase =~ /.*virtual.*touch.*/) + + # update attributes + nb.passerelle = @passerelle + nb.reference = ref + nb.bien_type = cat + nb.bien_transaction = transaction_type + nb.bien_emplacement = loc + + nb.nb_piece = cat_root['NBRE_PIECES'].to_i + nb.nb_chambre = cat_root['NBRE_CHAMBRES'].to_i + nb.surface = cat_root['SURFACE_HABITABLE'].to_i + nb.surface_terrain = cat_root['SURFACE_TERRAIN'].to_i + nb.titre = b['INTITULE']['FR'] + + nb.prix = price + nb.description = b['COMMENTAIRES']['FR'] + nb.valeur_dpe = cat_root['CONSO_ANNUEL_ENERGIE'] + nb.classe_dpe = cat_root['CONSOMMATIONENERGETIQUE'] + nb.valeur_ges = cat_root['VALEUR_GES'] + nb.class_ges = cat_root['GAZEFFETDESERRE'] + nb.statut = 'new' + nb.save! + + # If new images : Drop old images, add current images + if b['IMAGES'] && b['IMAGES']['IMG'] + # un-attach old + nb.bien_photos.each{ |photo| + photo.bien = nil + photo.save! + } + pl = b['IMAGES']['IMG'] + + # When there only exists a single image, +pl+ will directly be the hash + pl = [pl] unless pl.kind_of? Array + # Map photo-hashes to medias, filter out failures, and add medias to good + number = 0 + pl.map { |p| import_remote_media(p,(number += 1),nb) } + end + nb.save! + + return true + + return + end + +end \ No newline at end of file