@@ -1,7 +1,5 @@ -v1.9.3. Move Postgres stored procedures into a migration generator (Lang Riley). - -v1.9.2. Add geodistance support (Jeremy Seitz, Mark Lane). +v1.10. Add geodistance support (Jeremy Seitz, Mark Lane). Move Postgres stored procedures into a migration generator (Lang Riley). Duplicate text :sortable columns so that they are also searchable. v1.9.1. Add ultrasphinx:index:merge task for index merging, and a note in DEPLOYMENT_NOTES about how to use it. CHANGELOG @@ -232,15 +232,9 @@ module Ultrasphinx def build_regular_fields(klass, fields, entries, column_strings, join_strings, group_bys, remaining_columns) entries.to_a.each do |entry| - - source_string = if entry['sortable'] - entry['field'] # Use the alias - else - "#{entry['table_alias']}.#{entry['field']}" # Use the column - end - + source_string = "#{entry['table_alias']}.#{entry['field']}" group_bys << source_string - column_strings, remaining_columns = install_field(fields, source_string, entry['as'], entry['function_sql'], entry['facet'], column_strings, remaining_columns) + column_strings, remaining_columns = install_field(fields, source_string, entry['as'], entry['function_sql'], entry['facet'], entry['sortable'], column_strings, remaining_columns) end [column_strings, join_strings, group_bys, remaining_columns] @@ -272,7 +266,7 @@ module Ultrasphinx source_string = "#{entry['table_alias']}.#{entry['field']}" group_bys << source_string - column_strings, remaining_columns = install_field(fields, source_string, entry['as'], entry['function_sql'], entry['facet'], column_strings, remaining_columns) + column_strings, remaining_columns = install_field(fields, source_string, entry['as'], entry['function_sql'], entry['facet'], entry['sortable'], column_strings, remaining_columns) end [column_strings, join_strings, group_bys, remaining_columns] @@ -305,7 +299,7 @@ module Ultrasphinx source_string = SQL_FUNCTIONS[ADAPTER]['group_concat']._interpolate(source_string, order_string) use_distinct = true - column_strings, remaining_columns = install_field(fields, source_string, entry['as'], entry['function_sql'], entry['facet'], column_strings, remaining_columns) + column_strings, remaining_columns = install_field(fields, source_string, entry['as'], entry['function_sql'], entry['facet'], entry['sortable'], column_strings, remaining_columns) elsif entry['fields'] # Regular concats @@ -315,7 +309,7 @@ module Ultrasphinx group_bys << subsource_string end.join(', ') + ")" - column_strings, remaining_columns = install_field(fields, source_string, entry['as'], entry['function_sql'], entry['facet'], column_strings, remaining_columns) + column_strings, remaining_columns = install_field(fields, source_string, entry['as'], entry['function_sql'], entry['facet'], entry['sortable'], column_strings, remaining_columns) else raise ConfigurationError, "Invalid concatenate parameters for #{model}: #{entry.inspect}." @@ -337,12 +331,18 @@ module Ultrasphinx end - def install_field(fields, source_string, as, function_sql, with_facet, column_strings, remaining_columns) + def install_field(fields, source_string, as, function_sql, with_facet, with_sortable, column_strings, remaining_columns) source_string = function_sql._interpolate(source_string) if function_sql column_strings << fields.cast(source_string, as) remaining_columns.delete(as) + # Generate duplicate text fields for sorting + if with_sortable + column_strings << fields.cast(source_string, "#{as}_sortable") + remaining_columns.delete("#{as}_sortable") + end + # Generate hashed integer fields for text grouping if with_facet column_strings << "#{SQL_FUNCTIONS[ADAPTER]['hash']._interpolate(source_string)} AS #{as}_facet" lib/ultrasphinx/configure.rb @@ -107,8 +107,8 @@ This is a special singleton configuration class that stores the index field conf # XXX I think this is here for migrations Ultrasphinx.say "warning: field #{entry['field']} is not present in #{model}" else - save_and_verify_type(entry['as'], klass.columns_hash[entry['field']].type, entry['sortable'], klass) - install_facets!(entry, klass) + save_and_verify_type(entry['as'], klass.columns_hash[entry['field']].type, nil, klass) + install_duplicate_fields!(entry, klass) end end @@ -119,15 +119,15 @@ This is a special singleton configuration class that stores the index field conf association_model = get_association_model(klass, entry) - save_and_verify_type(entry['as'] || entry['field'], association_model.columns_hash[entry['field']].type, entry['sortable'], klass) - install_facets!(entry, klass) + save_and_verify_type(entry['as'] || entry['field'], association_model.columns_hash[entry['field']].type, nil, klass) + install_duplicate_fields!(entry, klass) end # Regular concats are CHAR, group_concats are BLOB and need to be cast to CHAR options['concatenate'].to_a.each do |entry| extract_table_alias!(entry, klass) - save_and_verify_type(entry['as'], 'text', entry['sortable'], klass) - install_facets!(entry, klass) + save_and_verify_type(entry['as'], 'text', nil, klass) + install_duplicate_fields!(entry, klass) end rescue ActiveRecord::StatementInvalid @@ -138,11 +138,15 @@ This is a special singleton configuration class that stores the index field conf self end - def install_facets!(entry, klass) + def install_duplicate_fields!(entry, klass) if entry['facet'] save_and_verify_type(entry['as'], 'text', nil, klass) # source must be a string save_and_verify_type("#{entry['as']}_facet", 'integer', nil, klass) end + if entry['sortable'] + save_and_verify_type(entry['as'], 'text', nil, klass) # source must be a string + save_and_verify_type("#{entry['as']}_sortable", 'text', true, klass) + end entry end lib/ultrasphinx/fields.rb @@ -220,19 +220,7 @@ If the associations weren't just <tt>has_many</tt> and <tt>belongs_to</tt>, you entry._stringify_all!('sortable', 'facet') entry.assert_valid_keys ['class_name', 'association_name', 'field', 'as', 'association_sql', 'facet', 'function_sql', 'sortable'] end - - # Add special alias columns for text sortables so that the original column doesn't get - # removed, since Sphinx will either store stems or ordinals but not both. We use the - # original alias as source so that the query doesn't get repeated in case it is - # resource-intensive. - opts.slice('fields', 'concatenate', 'include').values._flatten_once.each do |entry| - if entry['sortable'] - entry.delete('sortable') - name = "#{entry['as'] || entry['field']}" - opts['fields'] << {'field' => name, 'as' => "#{name}_sortable", 'sortable' => true} - end - end - + Ultrasphinx::MODEL_CONFIGURATION[self.name] = opts end end lib/ultrasphinx/is_indexed.rb @@ -1,5 +1,5 @@ -# Auto-generated at Mon Mar 31 00:50:43 -0400 2008. +# Auto-generated at Mon Mar 31 02:37:26 -0400 2008. # Hand modifications will be overwritten. # /Users/eweaver/Desktop/projects/ub/vendor/plugins/ultrasphinx/test/integration/app/config/ultrasphinx/default.base indexer { @@ -70,7 +70,7 @@ sql_host = localhost sql_pass = sql_user = root sql_query_range = SELECT MIN(id) , MAX(id) FROM sellers -sql_query = SELECT (sellers.id * 5 + 1) AS id, '' AS address_name, sellers.capitalization AS capitalization, 'Seller' AS class, 1 AS class_id, '' AS company, 0 AS company_facet, sellers.company_name AS company_name, CAST(CRC32(sellers.company_name) AS unsigned) AS company_name_facet, company_name AS company_name_sortable, '' AS company_two, 0 AS company_two_facet, '' AS content, UNIX_TIMESTAMP(sellers.created_at) AS created_at, 0 AS deleted, '' AS email, 0 AS lat, 0 AS lng, '' AS login, sellers.mission_statement AS mission_statement, mission_statement AS mission_statement_sortable, '' AS name, '' AS state, sellers.user_id AS user_id FROM sellers WHERE sellers.id >= $start AND sellers.id <= $end GROUP BY sellers.id +sql_query = SELECT (sellers.id * 5 + 1) AS id, '' AS address_name, sellers.capitalization AS capitalization, 'Seller' AS class, 1 AS class_id, '' AS company, 0 AS company_facet, sellers.company_name AS company_name, CAST(CRC32(sellers.company_name) AS unsigned) AS company_name_facet, sellers.company_name AS company_name_sortable, '' AS company_two, 0 AS company_two_facet, '' AS content, UNIX_TIMESTAMP(sellers.created_at) AS created_at, 0 AS deleted, '' AS email, 0 AS lat, 0 AS lng, '' AS login, sellers.mission_statement AS mission_statement, sellers.mission_statement AS mission_statement_sortable, '' AS name, '' AS state, sellers.user_id AS user_id FROM sellers WHERE sellers.id >= $start AND sellers.id <= $end GROUP BY sellers.id sql_attr_float = capitalization sql_attr_uint = class_id @@ -233,7 +233,7 @@ sql_host = localhost sql_pass = sql_user = root sql_query_range = SELECT MIN(id) , MAX(id) FROM sellers WHERE sellers.updated_at > DATE_SUB(NOW(), INTERVAL 88200 SECOND) -sql_query = SELECT (sellers.id * 5 + 1) AS id, '' AS address_name, sellers.capitalization AS capitalization, 'Seller' AS class, 1 AS class_id, '' AS company, 0 AS company_facet, sellers.company_name AS company_name, CAST(CRC32(sellers.company_name) AS unsigned) AS company_name_facet, company_name AS company_name_sortable, '' AS company_two, 0 AS company_two_facet, '' AS content, UNIX_TIMESTAMP(sellers.created_at) AS created_at, 0 AS deleted, '' AS email, 0 AS lat, 0 AS lng, '' AS login, sellers.mission_statement AS mission_statement, mission_statement AS mission_statement_sortable, '' AS name, '' AS state, sellers.user_id AS user_id FROM sellers WHERE sellers.id >= $start AND sellers.id <= $end AND sellers.updated_at > DATE_SUB(NOW(), INTERVAL 88200 SECOND) GROUP BY sellers.id +sql_query = SELECT (sellers.id * 5 + 1) AS id, '' AS address_name, sellers.capitalization AS capitalization, 'Seller' AS class, 1 AS class_id, '' AS company, 0 AS company_facet, sellers.company_name AS company_name, CAST(CRC32(sellers.company_name) AS unsigned) AS company_name_facet, sellers.company_name AS company_name_sortable, '' AS company_two, 0 AS company_two_facet, '' AS content, UNIX_TIMESTAMP(sellers.created_at) AS created_at, 0 AS deleted, '' AS email, 0 AS lat, 0 AS lng, '' AS login, sellers.mission_statement AS mission_statement, sellers.mission_statement AS mission_statement_sortable, '' AS name, '' AS state, sellers.user_id AS user_id FROM sellers WHERE sellers.id >= $start AND sellers.id <= $end AND sellers.updated_at > DATE_SUB(NOW(), INTERVAL 88200 SECOND) GROUP BY sellers.id sql_attr_float = capitalization sql_attr_uint = class_id test/integration/app/config/ultrasphinx/development.conf.canonical 69d9f5cab84753d0778a2460be3d17cf866419a9 Evan Weaver evan@cloudbur.st http://github.com/fauna/ultrasphinx/commit/9b5b7599d5f0ad2bb30beb346bcc33c56ae95cf8 9b5b7599d5f0ad2bb30beb346bcc33c56ae95cf8 2008-03-30T23:39:23-07:00 2008-03-30T23:39:23-07:00 Handle the sortable duplicates the same way as the facet duplicates. 3b8bac59f0986cbf6893551ea209ac34bb04ef66 Evan Weaver evan@cloudbur.st