Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 171 lines (161 sloc) 5.314 kb
628522ec »
2007-12-29 sha1-lookup: more memory efficient search in sorted list of SHA-1
1 #include "cache.h"
2 #include "sha1-lookup.h"
3
4 /*
5 * Conventional binary search loop looks like this:
6 *
7 * unsigned lo, hi;
8 * do {
9 * unsigned mi = (lo + hi) / 2;
10 * int cmp = "entry pointed at by mi" minus "target";
11 * if (!cmp)
12 * return (mi is the wanted one)
13 * if (cmp > 0)
14 * hi = mi; "mi is larger than target"
15 * else
16 * lo = mi+1; "mi is smaller than target"
17 * } while (lo < hi);
18 *
19 * The invariants are:
20 *
21 * - When entering the loop, lo points at a slot that is never
22 * above the target (it could be at the target), hi points at a
23 * slot that is guaranteed to be above the target (it can never
24 * be at the target).
25 *
26 * - We find a point 'mi' between lo and hi (mi could be the same
27 * as lo, but never can be as same as hi), and check if it hits
28 * the target. There are three cases:
29 *
30 * - if it is a hit, we are happy.
31 *
32 * - if it is strictly higher than the target, we set it to hi,
33 * and repeat the search.
34 *
35 * - if it is strictly lower than the target, we update lo to
36 * one slot after it, because we allow lo to be at the target.
37 *
38 * If the loop exits, there is no matching entry.
39 *
40 * When choosing 'mi', we do not have to take the "middle" but
41 * anywhere in between lo and hi, as long as lo <= mi < hi is
42 * satisfied. When we somehow know that the distance between the
43 * target and lo is much shorter than the target and hi, we could
44 * pick mi that is much closer to lo than the midway.
45 *
46 * Now, we can take advantage of the fact that SHA-1 is a good hash
47 * function, and as long as there are enough entries in the table, we
48 * can expect uniform distribution. An entry that begins with for
49 * example "deadbeef..." is much likely to appear much later than in
50 * the midway of the table. It can reasonably be expected to be near
51 * 87% (222/256) from the top of the table.
52 *
12ecb011 »
2007-12-30 sha1-lookup: make selection of 'middle' less aggressive
53 * However, we do not want to pick "mi" too precisely. If the entry at
54 * the 87% in the above example turns out to be higher than the target
55 * we are looking for, we would end up narrowing the search space down
56 * only by 13%, instead of 50% we would get if we did a simple binary
57 * search. So we would want to hedge our bets by being less aggressive.
58 *
628522ec »
2007-12-29 sha1-lookup: more memory efficient search in sorted list of SHA-1
59 * The table at "table" holds at least "nr" entries of "elem_size"
60 * bytes each. Each entry has the SHA-1 key at "key_offset". The
61 * table is sorted by the SHA-1 key of the entries. The caller wants
62 * to find the entry with "key", and knows that the entry at "lo" is
63 * not higher than the entry it is looking for, and that the entry at
64 * "hi" is higher than the entry it is looking for.
65 */
66 int sha1_entry_pos(const void *table,
67 size_t elem_size,
68 size_t key_offset,
69 unsigned lo, unsigned hi, unsigned nr,
70 const unsigned char *key)
71 {
72 const unsigned char *base = table;
73 const unsigned char *hi_key, *lo_key;
74 unsigned ofs_0;
75 static int debug_lookup = -1;
76
77 if (debug_lookup < 0)
78 debug_lookup = !!getenv("GIT_DEBUG_LOOKUP");
79
80 if (!nr || lo >= hi)
81 return -1;
82
83 if (nr == hi)
84 hi_key = NULL;
85 else
86 hi_key = base + elem_size * hi + key_offset;
87 lo_key = base + elem_size * lo + key_offset;
88
89 ofs_0 = 0;
90 do {
91 int cmp;
92 unsigned ofs, mi, range;
93 unsigned lov, hiv, kyv;
94 const unsigned char *mi_key;
95
96 range = hi - lo;
97 if (hi_key) {
98 for (ofs = ofs_0; ofs < 20; ofs++)
99 if (lo_key[ofs] != hi_key[ofs])
100 break;
101 ofs_0 = ofs;
102 /*
103 * byte 0 thru (ofs-1) are the same between
104 * lo and hi; ofs is the first byte that is
105 * different.
106 */
107 hiv = hi_key[ofs_0];
108 if (ofs_0 < 19)
109 hiv = (hiv << 8) | hi_key[ofs_0+1];
110 } else {
111 hiv = 256;
112 if (ofs_0 < 19)
113 hiv <<= 8;
114 }
115 lov = lo_key[ofs_0];
116 kyv = key[ofs_0];
117 if (ofs_0 < 19) {
118 lov = (lov << 8) | lo_key[ofs_0+1];
119 kyv = (kyv << 8) | key[ofs_0+1];
120 }
121 assert(lov < hiv);
122
123 if (kyv < lov)
124 return -1 - lo;
125 if (hiv < kyv)
126 return -1 - hi;
127
12ecb011 »
2007-12-30 sha1-lookup: make selection of 'middle' less aggressive
128 /*
129 * Even if we know the target is much closer to 'hi'
130 * than 'lo', if we pick too precisely and overshoot
131 * (e.g. when we know 'mi' is closer to 'hi' than to
132 * 'lo', pick 'mi' that is higher than the target), we
133 * end up narrowing the search space by a smaller
134 * amount (i.e. the distance between 'mi' and 'hi')
135 * than what we would have (i.e. about half of 'lo'
136 * and 'hi'). Hedge our bets to pick 'mi' less
137 * aggressively, i.e. make 'mi' a bit closer to the
138 * middle than we would otherwise pick.
139 */
140 kyv = (kyv * 6 + lov + hiv) / 8;
141 if (lov < hiv - 1) {
142 if (kyv == lov)
143 kyv++;
144 else if (kyv == hiv)
145 kyv--;
146 }
628522ec »
2007-12-29 sha1-lookup: more memory efficient search in sorted list of SHA-1
147 mi = (range - 1) * (kyv - lov) / (hiv - lov) + lo;
148
149 if (debug_lookup) {
150 printf("lo %u hi %u rg %u mi %u ", lo, hi, range, mi);
151 printf("ofs %u lov %x, hiv %x, kyv %x\n",
152 ofs_0, lov, hiv, kyv);
153 }
154 if (!(lo <= mi && mi < hi))
155 die("assertion failure lo %u mi %u hi %u %s",
156 lo, mi, hi, sha1_to_hex(key));
157
158 mi_key = base + elem_size * mi + key_offset;
159 cmp = memcmp(mi_key + ofs_0, key + ofs_0, 20 - ofs_0);
160 if (!cmp)
161 return mi;
162 if (cmp > 0) {
163 hi = mi;
164 hi_key = mi_key;
12ecb011 »
2007-12-30 sha1-lookup: make selection of 'middle' less aggressive
165 } else {
628522ec »
2007-12-29 sha1-lookup: more memory efficient search in sorted list of SHA-1
166 lo = mi + 1;
167 lo_key = mi_key + elem_size;
168 }
169 } while (lo < hi);
170 return -lo-1;
171 }
Something went wrong with that request. Please try again.