Skip to content
This repository

fix LC_COLLATE issue #176 #799

Open
wants to merge 1 commit into from

3 participants

charsyam Baina Deval openbaas
charsyam

sortCompare has a bug in sort.c

if server.sort_alpha is true, sort_alpha has nothing to do with sort_bypattern.

so it has to compare with strcoll instead of compareStringObject.

and add locale config parameter to set locale easily.

Baina Deval

The Redis server itself stores all data as a binary objects, so it is not dependent on the encoding. The server will just store what is sent by the client (including UTF-8 chars).

Here are a few experiments:

$ echo téléphone | hexdump -C
00000000 74 c3 a9 6c c3 a9 70 68 6f 6e 65 0a |t..l..phone.|
c3a9 is the representation of the 'é' char.

$ redis-cli

set t téléphone
OK
get t
"t\xc3\xa9l\xc3\xa9phone"
Actually the data is correctly stored in the Redis server. However, when it is launched in a terminal, the Redis client interprets the output and applies the sdscatrepr function to transform non printable chars (whose definition is locale dependent, and may be broken for multibyte chars anyway).

A simple workaround is to launch redis-cli with the 'raw' option:

$ redis-cli --raw

get t
téléphone
Your own application will probably use one of the client libraries rather than redis-cli, so it should not be a problem in practice.

charsyam

@bania. I don't think so. of course, redis server itself stores all data as a binary. but this problem is not about just storing.. for example. in Polish
redis> LPUSH test adam
(integer) 1
redis> LPUSH test łukasz
(integer) 2
redis> LPUSH test zuzanna
(integer) 3

redis> SORT test ALPHA
1) "adam"
2) "zuzanna"
3) "\xc5\x82ukasz"

łukasz has to be ahead of zuzanna.
but SORT result is wrong. we expect below result.

redis> SORT test ALPHA
1) "adam"
2) "\xc5\x82ukasz"
3) "zuzanna"

and my patch is about this problem :) Thank you.

Baina Deval

@charsyam You're right! Does your patch support redis hash sort(sorting by hash values)?
Thank you.

charsyam

@baina I don't know. I just found miss cases of sortCompare in sortCommand.

charsyam

@antirez I think this issue is obviously bug in special environment to use special letters.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Showing 1 unique commit by 1 author.

Nov 29, 2012
openbaas openbaas fix LC_COLLATE issue #176 5c9a4ad
This page is out of date. Refresh to see the latest.
2  src/config.c
@@ -189,6 +189,8 @@ void loadServerConfigFromString(char *config) {
189 189 if (server.maxclients < 1) {
190 190 err = "Invalid max clients limit"; goto loaderr;
191 191 }
  192 + } else if (!strcasecmp(argv[0],"locale") && argc == 2) {
  193 + server.locale = zstrdup(argv[1]);
192 194 } else if (!strcasecmp(argv[0],"maxmemory") && argc == 2) {
193 195 server.maxmemory = memtoll(argv[1],NULL);
194 196 } else if (!strcasecmp(argv[0],"maxmemory-policy") && argc == 2) {
7 src/redis.c
@@ -49,6 +49,8 @@
49 49 #include <math.h>
50 50 #include <sys/resource.h>
51 51 #include <sys/utsname.h>
  52 +#include <locale.h>
  53 +#include <stdlib.h>
52 54
53 55 /* Our shared "common" objects */
54 56
@@ -1235,6 +1237,10 @@ void initServerConfig() {
1235 1237 server.assert_line = 0;
1236 1238 server.bug_report_start = 0;
1237 1239 server.watchdog_period = 0;
  1240 + server.locale = getenv("LC_COLLATE");
  1241 + if( server.locale == NULL ){
  1242 + server.locale = zstrdup("en_US.utf8");
  1243 + }
1238 1244 }
1239 1245
1240 1246 /* This function will try to raise the max number of open files accordingly to
@@ -2672,6 +2678,7 @@ int main(int argc, char **argv) {
2672 2678 if (server.daemonize) createPidFile();
2673 2679 redisAsciiArt();
2674 2680
  2681 + setlocale( LC_COLLATE, server.locale );
2675 2682 if (!server.sentinel_mode) {
2676 2683 /* Things only needed when not running in Sentinel mode. */
2677 2684 redisLog(REDIS_WARNING,"Server started, Redis version " REDIS_VERSION);
2  src/redis.h
@@ -796,6 +796,8 @@ struct redisServer {
796 796 int assert_line;
797 797 int bug_report_start; /* True if bug report header was already logged. */
798 798 int watchdog_period; /* Software watchdog period in ms. 0 = off */
  799 + /* locale */
  800 + char *locale;
799 801 };
800 802
801 803 typedef struct pubsubPattern {
22 src/sort.c
@@ -135,6 +135,26 @@ robj *lookupKeyByPattern(redisDb *db, robj *pattern, robj *subst) {
135 135 /* sortCompare() is used by qsort in sortCommand(). Given that qsort_r with
136 136 * the additional parameter is not standard but a BSD-specific we have to
137 137 * pass sorting parameters via the global 'server' structure */
  138 +int compareStringWithLocale(robj *a, robj *b) {
  139 + redisAssertWithInfo(NULL,a,a->type == REDIS_STRING && b->type == REDIS_STRING);
  140 + char bufa[128], bufb[128], *astr, *bstr;
  141 +
  142 + if (a == b) return 0;
  143 + if (a->encoding != REDIS_ENCODING_RAW) {
  144 + ll2string(bufa,sizeof(bufa),(long) a->ptr);
  145 + astr = bufa;
  146 + } else {
  147 + astr = a->ptr;
  148 + }
  149 + if (b->encoding != REDIS_ENCODING_RAW) {
  150 + ll2string(bufb,sizeof(bufb),(long) b->ptr);
  151 + bstr = bufb;
  152 + } else {
  153 + bstr = b->ptr;
  154 + }
  155 + return strcoll(astr,bstr);
  156 +}
  157 +
138 158 int sortCompare(const void *s1, const void *s2) {
139 159 const redisSortObject *so1 = s1, *so2 = s2;
140 160 int cmp;
@@ -168,7 +188,7 @@ int sortCompare(const void *s1, const void *s2) {
168 188 }
169 189 } else {
170 190 /* Compare elements directly. */
171   - cmp = compareStringObjects(so1->obj,so2->obj);
  191 + cmp = compareStringWithLocale(so1->obj,so2->obj);
172 192 }
173 193 }
174 194 return server.sort_desc ? -cmp : cmp;

Tip: You can add notes to lines in a file. Hover to the left of a line to make a note

Something went wrong with that request. Please try again.