Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
  • 11 commits
  • 7 files changed
  • 0 comments
  • 1 contributor
22  lib/anlas_import.rb
... ...
@@ -1,21 +1,17 @@
1 1
 # encoding: utf-8
2 2
 module AnlasImport
3 3
 
4  
-  PROC_NAME = "anlas_import_xml"
  4
+  class Base
5 5
 
6  
-  class << self
  6
+    def self.backup_dir
  7
+      "/home/webmaster/backups/imports/"
  8
+    end # backup_dir
7 9
 
8  
-    def can_start?
  10
+    def self.run
  11
+      puts "[AnlasImport::Base] Method `run` must be overwrited."
  12
+    end # self.run
9 13
 
10  
-      return false if defined?(::IRB)
11  
-      return false if defined?(::Rake)
12  
-      return false unless defined?(::Rails)
13  
-      return false if ::Rails.env.to_s != "production"
14  
-      true
15  
-
16  
-    end # can_start?
17  
-
18  
-  end # class << self
  14
+  end # Base
19 15
 
20 16
 end # AnlasImport
21 17
 
@@ -30,8 +26,6 @@ def can_start?
30 26
 require 'anlas_import/ext'
31 27
 require 'anlas_import/mailer'
32 28
 
33  
-require 'anlas_import/base'
34  
-
35 29
 require 'anlas_import/xml_parser'
36 30
 require 'anlas_import/worker'
37 31
 require 'anlas_import/manager'
40  lib/anlas_import/base.rb
... ...
@@ -1,40 +0,0 @@
1  
-# encoding: utf-8
2  
-module AnlasImport
3  
-  
4  
-  class Base
5  
-
6  
-    class << self
7  
-
8  
-      def run
9  
-
10  
-        ::Imp.run(::AnlasImport::PROC_NAME, ::File.join(::Rails.root, "log", "import_xml.log")) do
11  
-          
12  
-          puts "#{::Time.now} #{::AnlasImport::PROC_NAME} start"
13  
-          
14  
-          import_dir    = "/home/import"
15  
-          log_dir       = ::File.join("#{::Rails.root}", "log")
16  
-          db_conf_file  = ::File.join("#{::Rails.root}", "config", "mongoid.yml")
17  
-
18  
-          loop do
19  
-            
20  
-            ::AnlasImport::Manager.new(import_dir, log_dir, db_conf_file).run do |inserted, updated|
21  
-
22  
-              updated.each do |id|
23  
-                ::Item.where(:_id => id).first.try(&:update_sphinx)
24  
-              end
25  
-              
26  
-            end # run
27  
-
28  
-            sleep 5 * 60 # 5 minutes
29  
-
30  
-          end # loop
31  
-
32  
-        end # Imp.run
33  
-
34  
-      end # self.run
35  
-
36  
-    end # class << self
37  
-
38  
-  end # Base
39  
-  
40  
-end # ImportXml
55  lib/anlas_import/manager.rb
... ...
@@ -1,33 +1,31 @@
1 1
 # encoding: UTF-8
2 2
 module AnlasImport
3 3
 
4  
-  # Предварительная обработка выгрузки (распаковка архивов). Проверка
5  
-  # соедиения с базой данных. Запуск обработчика. Отправка отчетов.
  4
+  # Предварительная обработка выгрузки (распаковка архивов).
  5
+  # Запуск обработчика. Отправка отчетов.
6 6
   class Manager
7  
-    
8  
-    def initialize(import_dir, log_dir, db_conf_file)
  7
+
  8
+    def initialize(import_dir, log_dir)
9 9
 
10 10
       @config = {
11 11
         :dir => import_dir,
12  
-        :log => log_dir,
13  
-        :db_conf  => db_conf_file
  12
+        :log => log_dir
14 13
       }
15 14
 
16 15
       reset
17 16
       check_import_dir
18  
-      read_db_config
19 17
 
20 18
     end # new
21 19
 
22 20
     def run
23 21
 
24 22
       before
25  
-      
  23
+
26 24
       if @errors.empty?
27 25
 
28 26
         start = ::Time.now.to_i
29 27
         processing
30  
-        
  28
+
31 29
         if (@inserted_items.length + @updaed_items.length > 0)
32 30
 
33 31
           @errors << "[#{Time.now.strftime('%H:%M:%S %d-%m-%Y')}] Обработка файлов импорта ============================"
@@ -66,19 +64,13 @@ def reset
66 64
     end # reset
67 65
 
68 66
     def before
69  
-
70  
-      open_db_connection
71 67
       extract_zip_files
72  
-
73 68
     end # before_start
74 69
 
75 70
     def after
76 71
 
77  
-      #Item.where(:marking_of_goods.in => @inserted_items).map(&:init_features)
78  
-
79 72
       self.log(@errors.flatten.join("\n")) unless @errors.empty?
80 73
       close_logger
81  
-      close_db_connect
82 74
 
83 75
     end # after
84 76
 
@@ -91,7 +83,7 @@ def processing
91 83
 
92 84
       files.each do |xml_file|
93 85
 
94  
-        worker = ::AnlasImport::Worker.new(xml_file, @conn).parse
  86
+        worker = ::AnlasImport::Worker.new(xml_file).parse
95 87
 
96 88
         @errors << worker.errors
97 89
         @inserted_items = @inserted_items.concat(worker.inserted)
@@ -114,29 +106,6 @@ def check_import_dir
114 106
 
115 107
     end # check_import_dir
116 108
 
117  
-    def read_db_config
118  
-
119  
-      begin
120  
-        @config[:db_conf] = ::YAML::load_file(@config[:db_conf])["production"]
121  
-      rescue => e
122  
-        @errors << "#{e}"
123  
-      end
124  
-
125  
-      @errors << "Не найден раздел :production в файле конфигурации database.yml " if @config[:db_conf].nil?
126  
-
127  
-    end # read_db_config
128  
-
129  
-    def open_db_connection
130  
-
131  
-      begin
132  
-        ::Mongoid.database.collection("admin").find_one
133  
-        @conn ||= ::Mongoid.database
134  
-      rescue => e
135  
-        @errors << "Нет соединения с базой данных."
136  
-      end # begin
137  
-      
138  
-    end # open_db_connection
139  
-
140 109
     def extract_zip_files
141 110
 
142 111
       # Ищем и распаковываем все zip-архивы, после - удаляем
@@ -155,7 +124,7 @@ def extract_zip_files
155 124
               ::FileUtils.rm_rf f_path if ::File.exist?(f_path)
156 125
               ::FileUtils.mkdir_p(::File.dirname(f_path))
157 126
               zip_file.extract(f, f_path)
158  
-              
  127
+
159 128
             } # each
160 129
 
161 130
           } # open
@@ -163,7 +132,7 @@ def extract_zip_files
163 132
           ::FileUtils.rm_rf(zip)
164 133
 
165 134
         rescue
166  
-        end  
  135
+        end
167 136
 
168 137
       end # Dir.glob
169 138
 
@@ -191,10 +160,6 @@ def close_logger
191 160
 
192 161
     end # close_logger
193 162
 
194  
-    def close_db_connect
195  
-      # no need for mongoid
196  
-    end # close_db_connect
197  
-
198 163
   end # Manager
199 164
 
200 165
 end # AnlasImport
5  lib/anlas_import/railtie.rb
... ...
@@ -1,13 +1,12 @@
1  
-# encoding: utf-8
2 1
 require 'rails/railtie'
3 2
 
4 3
 module AnlasImport
5 4
 
6 5
   class Railtie < ::Rails::Railtie #:nodoc:
7 6
 
8  
-    initializer 'anlas_import' do |app|
  7
+    config.after_initialize do
9 8
 
10  
-      ::AnlasImport::Base.run if ::AnlasImport.can_start?
  9
+      ::AnlasImport::Base.run
11 10
 
12 11
     end # initializer
13 12
 
2  lib/anlas_import/version.rb
... ...
@@ -1,6 +1,6 @@
1 1
 # encoding: utf-8
2 2
 module AnlasImport
3 3
 
4  
-  VERSION = '0.3.2'
  4
+  VERSION = '0.4.1'
5 5
 
6 6
 end # AnlasImport
208  lib/anlas_import/worker.rb
... ...
@@ -1,34 +1,19 @@
1 1
 # encoding: UTF-8
2 2
 module AnlasImport
3 3
 
4  
-  BACKUP_DIR = "/home/webmaster/backups/imports/"
5  
-
6 4
   # Сохранение данных (добавление новых, обновление сущестующих), полученных
7 5
   # при разборе xml-файла.
8 6
   class Worker
9 7
 
10  
-    MOG_EQ = {
11  
-      'g' => 'a',
12  
-      'v' => 'v',
13  
-      'h' => 'h',
14  
-      'a' => '',
15  
-      'e' => 'e',
16  
-      't' => 't',
17  
-      'z' => 'z',
18  
-      'i' => 'i'
19  
-    }
20  
-
21  
-    def initialize(file, conn)
  8
+    def initialize(file)
22 9
 
23  
-      @errors, @ins, @upd = [], [], []
24  
-      @file, @conn = file, conn
  10
+      @catalogs = {}
25 11
 
  12
+      @errors, @ins, @upd, @file = [], [], [], file
26 13
       @file_name = ::File.basename(@file)
27 14
 
28 15
       unless @file && ::FileTest.exists?(@file)
29 16
         @errors << "Файл не найден: #{@file}"
30  
-      else
31  
-        @errors << "Не могу соединиться с базой данных!" unless @conn
32 17
       end # unless
33 18
 
34 19
     end # new
@@ -54,7 +39,7 @@ def inserted
54 39
 
55 40
     private
56 41
 
57  
-    def init_saver(catalog)
  42
+    def init_saver
58 43
 
59 44
       # Блок сохраниения данных в базу
60 45
       @saver = lambda { |artikul, artikulprod, name, purchasing_price, available, gtd_number, storehouse|
@@ -62,35 +47,30 @@ def init_saver(catalog)
62 47
         name        = clear_name(name).strip.escape
63 48
         artikul     = artikul.strip.escape
64 49
         artikulprod = artikulprod.strip.escape
  50
+        postfix     = artikul[-1] || ""
65 51
 
66 52
         # Проверка товара на наличие букв "яя" вначле названия (такие товары не выгружаем)
67 53
         unless skip_by_name(name)
68 54
 
69  
-          orig_artikul = artikul
70  
-          finded = target_exists(orig_artikul)
71  
-
72  
-          #unless finded
73  
-            #postfix = artikul[/[gvhaetzi]$/]
74  
-            #if postfix
75  
-              #artikul = orig_artikul.sub(/[gvhaetzi]$/, '')
76  
-              #artikul = MOG_EQ[postfix] + artikul
77  
-              #finded  = target_exists(artikul)
78  
-            #end # if
79  
-          #end # unless
80  
-
81  
-          if finded
  55
+          if (item = find_item(artikul))
82 56
 
83  
-            if update(artikul, name, purchasing_price, available)
  57
+            if update(item, name, purchasing_price, available, gtd_number, storehouse)
84 58
               @upd << artikul
85 59
             end
86 60
 
87 61
           else
88 62
 
89  
-            if insert(artikul, artikulprod, name, purchasing_price, available, gtd_number, storehouse, catalog)
90  
-              @ins << artikul
  63
+            if (catalog = catalog_for_import(postfix))
  64
+
  65
+              if insert(catalog, artikul, artikulprod, name, purchasing_price, available, gtd_number, storehouse)
  66
+                @ins << artikul
  67
+              end
  68
+
  69
+            else
  70
+              @errors << "[Errors] Каталог выгрузки не найден. Товар: #{artikul} -> #{name} (postfix: #{postfix})"
91 71
             end
92 72
 
93  
-          end
  73
+          end # if
94 74
 
95 75
         end # unless
96 76
 
@@ -100,131 +80,113 @@ def init_saver(catalog)
100 80
 
101 81
     def work_with_file
102 82
 
103  
-      unless (catalog = catalog_for_import( prefix_file ))
104  
-       @errors << "Каталог выгрузки не найден! Файл: #{@file}"
105  
-      else
106  
-
107  
-        init_saver(catalog)
  83
+      init_saver
108 84
 
109  
-        pt = ::AnlasImport::XmlParser.new(@saver)
  85
+      pt = ::AnlasImport::XmlParser.new(@saver)
110 86
 
111  
-        parser = ::Nokogiri::XML::SAX::Parser.new(pt)
112  
-        parser.parse_file(@file)
  87
+      parser = ::Nokogiri::XML::SAX::Parser.new(pt)
  88
+      parser.parse_file(@file)
113 89
 
114  
-        unless (errors = pt.errors).empty?
115  
-          @errors << errors
116  
-        end
  90
+      unless (errors = pt.errors).empty?
  91
+        @errors << errors
  92
+      end
117 93
 
118  
-        begin
119  
-          ::FileUtils.mv(@file, AnlasImport::BACKUP_DIR)
120  
-        rescue SystemCallError
121  
-          puts "Не могу переместить файл `#{@file_name}` в `#{AnlasImport::BACKUP_DIR}`"
122  
-          ::FileUtils.rm_rf(@file)
123  
-        end
124  
-
125  
-      end # unless
  94
+      begin
  95
+        ::FileUtils.mv(@file, ::AnlasImport::Base.backup_dir)
  96
+      rescue SystemCallError
  97
+        puts "Не могу переместить файл `#{@file_name}` в `#{::AnlasImport::Base.backup_dir}`"
  98
+        ::FileUtils.rm_rf(@file)
  99
+      end
126 100
 
127 101
     end # work_with_file
128 102
 
129  
-    def catalog_for_import(prefix)
130  
-
131  
-      catalog_import = @conn.collection("catalogs").find_one({
132  
-        "import_prefix" => (prefix.blank? ? "_" : prefix)
133  
-      })
134  
-
135  
-      catalog_import ? catalog_import : false
136  
-
137  
-    end # catalog_for_import
138  
-
139  
-    def target_exists(marking_of_goods)
140  
-
141  
-      item = @conn.collection("items").find_one({
142  
-        "marking_of_goods" => marking_of_goods
143  
-      })
144  
-
145  
-      item ? item : false
146  
-
147  
-    end # target_exists
  103
+    def catalog_for_import(postfix)
148 104
 
149  
-    def insert(artikul, artikulprod, name, purchasing_price, available, gtd_number, storehouse, catalog)
  105
+      unless (catalog = @catalogs[postfix])
150 106
 
151  
-      doc = {
  107
+        catalog = ::Catalog.safely.where(:import => postfix).first
  108
+        @catalogs[postfix] = catalog if catalog
152 109
 
153  
-        "name_1c"         => name,
154  
-        "name"            => name,
155  
-        "meta_title"      => name,
156  
-        "unmanaged"       => true,
157  
-
158  
-        "purchasing_price"=> purchasing_price.to_i,
  110
+      end # unless
159 111
 
160  
-        "storehouse"      => storehouse,
161  
-        "gtd_number"      => gtd_number,
  112
+      catalog
162 113
 
163  
-        "marking_of_goods" => artikul,
164  
-        "available"       => available.to_i,
165  
-        "marking_of_goods_manufacturer" => artikulprod,
  114
+    end # catalog_for_import
166 115
 
167  
-        "imported_at"     => ::Time.now.utc,
168  
-        "created_at"      => ::Time.now.utc,
  116
+    def find_item(marking_of_goods)
  117
+      ::Item.where(:marking_of_goods => marking_of_goods).first
  118
+    end # find_item
169 119
 
170  
-        "catalog_id"      => catalog["_id"],
171  
-        "catalog_lft"     => catalog["lft"],
172  
-        "catalog_rgt"     => catalog["rgt"]
173  
-      }
  120
+    def insert(catalog, artikul, artikulprod, name, purchasing_price, available, gtd_number, storehouse)
174 121
 
175  
-      opts = { :safe => true }
  122
+      item = ::Item.new
176 123
 
177  
-      begin
  124
+      item.marking_of_goods = artikul
  125
+      item.marking_of_goods_manufacturer = artikulprod
178 126
 
179  
-        @conn.collection("admin").update(
180  
-          {:name => "Item_counter"}, {"$inc" => {:count => 1}}, {:upsert => true}
181  
-        )
  127
+      item.catalog_id = catalog.id
  128
+      item.name_1c    = name
  129
+      item.name       = name
  130
+      item.meta_title = name
  131
+      item.unmanaged  = true
182 132
 
183  
-        counter = @conn.collection("admin").find_one(:name => "Item_counter")
184  
-        doc["uri"] = (counter ? counter["count"] : 0)
  133
+      item.purchasing_price = purchasing_price
185 134
 
186  
-        @conn.collection("items").insert(doc, opts)
  135
+      item.available  = available
  136
+      item.gtd_number = gtd_number
  137
+      item.storehouse = storehouse
187 138
 
188  
-        return true
  139
+      item.imported_at  = ::Time.now.utc
189 140
 
190  
-      rescue => e
191  
-        @errors << "[INSERT: #{artikul}] #{e}"
  141
+      unless item.save(validate: false)
  142
+        @errors << "[INSERT: #{artikul}] #{item.errors.inspect}"
192 143
         return false
193  
-      end # begin
  144
+      end
194 145
 
195  
-    end # insert
  146
+      true
196 147
 
197  
-    def update(artikul, name, purchasing_price, available)
  148
+    end # insert
198 149
 
199  
-      selector = { "marking_of_goods" => artikul }
  150
+    def update(item, name, purchasing_price, available, gtd_number, storehouse)
200 151
 
201  
-      doc = {
202  
-        "name_1c"           => name,
203  
-        "purchasing_price"  => purchasing_price.to_i,
204  
-        "available"         => available.to_i,
205  
-        "imported_at"       => ::Time.now.utc
206  
-      }
  152
+      item.name_1c    = name
207 153
 
208  
-      opts = { :safe => true }
  154
+      item.purchasing_price = purchasing_price
209 155
 
210  
-      begin
  156
+      item.available  = available
  157
+      item.gtd_number = gtd_number
  158
+      item.storehouse = storehouse
211 159
 
212  
-        @conn.collection("items").update(selector, { "$set" => doc }, opts)
213  
-        return true
  160
+      item.imported_at  = ::Time.now.utc
214 161
 
215  
-      rescue => e
216  
-        @errors << "[UPDATE: #{artikul}] #{e}"
  162
+      unless item.save(validate: false)
  163
+        @errors << "[UPDATE: #{item.marking_of_goods}] #{item.errors.inspect}"
217 164
         return false
218  
-      end # begin
  165
+      end
  166
+
  167
+      true
219 168
 
220 169
     end # update
221 170
 
222 171
     def skip_by_name(name)
223  
-      (name =~ /^я{2,}/u) === 0
  172
+      (name =~ /\A\s*я{2,}/u) === 0
224 173
     end # skip_by_name
225 174
 
226 175
     def clear_name(name)
227  
-      name.sub(/\s{0,}\+\s{0,}подарок\!{0,}\z/i, "")
  176
+
  177
+      name
  178
+        .sub(/\A\s+/, "")
  179
+        .gsub(/\!{0,}\z/i, "")
  180
+        .gsub(/акция/i, "")
  181
+        .gsub(/подарок/i, "")
  182
+        .gsub(/не заказывать/i, "")
  183
+        .gsub(/снижена цена/i, "")
  184
+        .gsub(/выведены/i, "")
  185
+        .gsub(/\(\)/, "")
  186
+        .gsub(/\s{0,}\+\s{0,}\z/, "")
  187
+        .gsub(/(\s){2,}/, '\\1')
  188
+        .sub(/\s+\z/, "")
  189
+
228 190
     end # clear_name
229 191
 
230 192
     def prefix_file
2  lib/anlas_import/xml_parser.rb
@@ -73,7 +73,7 @@ def tag_nom(attrs)
73 73
         attrs["ostatok"] || 0,
74 74
 
75 75
         # gtd_number (s),
76  
-        attrs["number_GTD"] || "",
  76
+        (attrs["number_GTD"] || "").gsub(/\-/, ""),
77 77
 
78 78
         # storehouse (s)
79 79
         attrs["sklad"] || ""

No commit comments for this range

Something went wrong with that request. Please try again.