Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

patches for tor on constrained devices + transparent torrc

  • Loading branch information...
commit ad84ee63ef2462b78a2571c7f3bb2789b5bb01f8 1 parent b267763
the grugq authored
118 feeds/packages/net/tor-alpha/Makefile
View
@@ -0,0 +1,118 @@
+#
+# Copyright (C) 2008-2012 OpenWrt.org
+#
+# This is free software, licensed under the GNU General Public License v2.
+# See /LICENSE for more information.
+#
+
+include $(TOPDIR)/rules.mk
+
+PKG_NAME:=tor-alpha
+PKG_VERSION:=0.2.3.22-rc
+PKG_RELEASE:=1
+
+PKG_SOURCE:=tor-$(PKG_VERSION).tar.gz
+PKG_SOURCE_URL:=https://www.torproject.org/dist \
+ https://archive.torproject.org/tor-package-archive
+PKG_MD5SUM:=c07431ee40a0e16dc1b4d1e8d266680c
+
+PKG_BUILD_DEPENDS:=libminiupnpc libnatpmp
+PKG_BUILD_DIR:=$(BUILD_DIR)/tor-$(PKG_VERSION)
+PKG_INSTALL:=1
+
+include $(INCLUDE_DIR)/package.mk
+
+define Package/tor-alpha/Default
+ SECTION:=net
+ CATEGORY:=Network
+ URL:=https://www.torproject.org/
+endef
+
+define Package/tor-alpha/Default/description
+ Tor is a toolset for a wide range of organizations and people that want to
+ improve their safety and security on the Internet. Using Tor can help you
+ anonymize web browsing and publishing, instant messaging, IRC, SSH, and
+ more. Tor also provides a platform on which software developers can build
+ new applications with built-in anonymity, safety, and privacy features.
+endef
+
+define Package/tor-alpha
+$(call Package/tor-alpha/Default)
+ TITLE:=An anonymous Internet communication system
+ DEPENDS:=+libevent2 +libevent2-openssl +libpthread +librt
+endef
+
+define Package/tor-alpha/description
+$(call Package/tor-alpha/Default/description)
+ This package contains the tor daemon.
+endef
+
+define Package/tor-alpha-fw-helper
+$(call Package/tor-alpha/Default)
+ TITLE:=Firewall helper for tor
+ DEPENDS:=+tor-alpha +libminiupnpc +libnatpmp
+endef
+
+define Package/tor-alpha-fw-helper/description
+$(call Package/tor-alpha/Default/description)
+ This package contains a helper for automatically configuring port forwarding.
+endef
+
+define Package/tor-alpha-geoip
+$(call Package/tor-alpha/Default)
+ TITLE:=GeoIP db for tor
+ DEPENDS:=+tor-alpha
+endef
+
+define Package/tor-alpha-geoip/description
+$(call Package/tor-alpha/Default/description)
+ This package contains a GeoIP database mapping IP addresses to countries.
+endef
+
+define Package/tor-alpha/conffiles
+/etc/tor/torrc
+endef
+
+CONFIGURE_ARGS += \
+ --with-libevent-dir="$(STAGING_DIR)/usr" \
+ --with-ssl-dir="$(STAGING_DIR)/usr" \
+ --enable-upnp \
+ --with-libminiupnpc-dir="$(STAGING_DIR)/usr" \
+ --enable-nat-pmp \
+ --with-libnatpmp-dir="$(STAGING_DIR)/usr" \
+ --disable-asciidoc
+
+ifneq ($(CONFIG_SSP_SUPPORT),y)
+ CONFIGURE_ARGS += \
+ --disable-gcc-hardening
+endif
+
+CONFIGURE_VARS += \
+ CROSS_COMPILE="yes"
+
+# pass CFLAGS again to override -O2 set by configure
+MAKE_FLAGS += \
+ CFLAGS="$(TARGET_CFLAGS)"
+
+define Package/tor-alpha/install
+ $(INSTALL_DIR) $(1)/usr/sbin
+ $(INSTALL_BIN) $(PKG_INSTALL_DIR)/usr/bin/tor $(1)/usr/sbin/
+ $(INSTALL_DIR) $(1)/etc/init.d
+ $(INSTALL_BIN) ./files/tor.init $(1)/etc/init.d/tor
+ $(INSTALL_DIR) $(1)/etc/tor
+ $(INSTALL_CONF) $(PKG_INSTALL_DIR)/etc/tor/torrc.sample $(1)/etc/tor/torrc
+endef
+
+define Package/tor-alpha-fw-helper/install
+ $(INSTALL_DIR) $(1)/usr/bin
+ $(INSTALL_BIN) $(PKG_INSTALL_DIR)/usr/bin/tor-fw-helper $(1)/usr/bin/
+endef
+
+define Package/tor-alpha-geoip/install
+ $(INSTALL_DIR) $(1)/usr/share/tor
+ $(CP) $(PKG_INSTALL_DIR)/usr/share/tor/geoip $(1)/usr/share/tor/
+endef
+
+$(eval $(call BuildPackage,tor-alpha))
+$(eval $(call BuildPackage,tor-alpha-fw-helper))
+$(eval $(call BuildPackage,tor-alpha-geoip))
30 feeds/packages/net/tor-alpha/files/tor.init
View
@@ -0,0 +1,30 @@
+#!/bin/sh /etc/rc.common
+# Copyright (C) 2006-2011 OpenWrt.org
+
+START=50
+STOP=50
+
+SERVICE_USE_PID=1
+
+start() {
+ user_exists tor 52 || user_add tor 52 52 /var/lib/tor
+ group_exists tor 52 || group_add tor 52
+ [ -f /var/run/tor.pid ] || {
+ touch /var/run/tor.pid
+ chown tor:tor /var/run/tor.pid
+ }
+ [ -d /var/lib/tor ] || {
+ mkdir -m 0755 -p /var/lib/tor
+ chmod 0700 /var/lib/tor
+ chown tor:tor /var/lib/tor
+ }
+ [ -d /var/log/tor ] || {
+ mkdir -m 0755 -p /var/log/tor
+ chown tor:tor /var/log/tor
+ }
+ service_start /usr/sbin/tor --PidFile /var/run/tor.pid
+}
+
+stop() {
+ service_stop /usr/sbin/tor
+}
919 feeds/packages/net/tor-alpha/patches/001-anon_mmap.patch
View
@@ -0,0 +1,919 @@
+diff -uNr tor-0.2.3.19-rc/src/common/util.c tor-0.2.3.19-rc-anon_mmap//src/common/util.c
+--- tor-0.2.3.19-rc/src/common/util.c 2012-07-06 16:32:28.000000000 -0400
++++ tor-0.2.3.19-rc-anon_mmap//src/common/util.c 2012-07-09 11:39:11.995224656 -0400
+@@ -73,6 +73,9 @@
+ #ifdef HAVE_SYS_FCNTL_H
+ #include <sys/fcntl.h>
+ #endif
++#ifdef HAVE_SYS_MMAN_H
++#include <sys/mman.h>
++#endif
+ #ifdef HAVE_TIME_H
+ #include <time.h>
+ #endif
+@@ -1994,9 +1997,7 @@
+ * we can write into a temporary file, and either remove the file on
+ * failure, or replace the original file on success. */
+ struct open_file_t {
+- char *tempname; /**< Name of the temporary file. */
+ char *filename; /**< Name of the original file. */
+- unsigned rename_on_close:1; /**< Are we using the temporary file or not? */
+ unsigned binary:1; /**< Did we open in binary mode? */
+ int fd; /**< fd for the open file. */
+ FILE *stdio_file; /**< stdio wrapper for <b>fd</b>. */
+@@ -2035,18 +2036,15 @@
+ #endif
+ new_file->fd = -1;
+ new_file->filename = tor_strdup(fname);
++ open_name = fname;
++
+ if (open_flags & O_APPEND) {
+- open_name = fname;
+- new_file->rename_on_close = 0;
+ append = 1;
+ open_flags &= ~O_APPEND;
+ } else {
+- tor_asprintf(&new_file->tempname, "%s.tmp", fname);
+- open_name = new_file->tempname;
+- /* We always replace an existing temporary file if there is one. */
++ /* We always replace an existing file if not appending. */
+ open_flags |= O_CREAT|O_TRUNC;
+ open_flags &= ~O_EXCL;
+- new_file->rename_on_close = 1;
+ }
+ if (open_flags & O_BINARY)
+ new_file->binary = 1;
+@@ -2074,7 +2072,6 @@
+ close(new_file->fd);
+ *data_out = NULL;
+ tor_free(new_file->filename);
+- tor_free(new_file->tempname);
+ tor_free(new_file);
+ return -1;
+ }
+@@ -2134,22 +2131,7 @@
+ abort_write = r = -1;
+ }
+
+- if (file_data->rename_on_close) {
+- tor_assert(file_data->tempname && file_data->filename);
+- if (abort_write) {
+- unlink(file_data->tempname);
+- } else {
+- tor_assert(strcmp(file_data->filename, file_data->tempname));
+- if (replace_file(file_data->tempname, file_data->filename)) {
+- log_warn(LD_FS, "Error replacing \"%s\": %s", file_data->filename,
+- strerror(errno));
+- r = -1;
+- }
+- }
+- }
+-
+ tor_free(file_data->filename);
+- tor_free(file_data->tempname);
+ tor_free(file_data);
+
+ return r;
+@@ -4552,3 +4534,166 @@
+ }
+ }
+
++/* =====
++ * Anonymous mmaps for holding large chunks of data
++ * ===== */
++
++/* Initial size of anonymous mmap and the increase step if we need
++ * to make it bigger */
++#define MMAP_INCREASE_SIZE (1<<18)
++
++/** Create a new anonymous memory mapping. Return 0 on success. */
++int
++tor_mmap_anon(tor_mmap_t *handle)
++{
++ char *map;
++ ssize_t map_size, page_size;
++
++ /* round up the initial map size to the next page boundary,
++ * and allocate an anonymous map */
++ page_size = getpagesize();
++ map_size = MMAP_INCREASE_SIZE + (MMAP_INCREASE_SIZE%page_size ? page_size-MMAP_INCREASE_SIZE%page_size : 0);
++
++ if (MAP_FAILED == (map = mmap(NULL, map_size, PROT_READ|PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0))) {
++ int save_errno = errno;
++ log_warn(LD_FS,"Could not create mmap: %s",
++ strerror(errno));
++ handle->data = NULL;
++ handle->size = handle->mapping_size = 0;
++ errno = save_errno;
++ return -1;
++ }
++
++ handle->data = map;
++ handle->size = 0;
++ handle->mapping_size = map_size;
++
++ return 0;
++}
++
++/* Free the mmap referred by the handle. Return 0. */
++int
++tor_munmap(tor_mmap_t *handle) {
++ munmap((void *)handle->data, handle->mapping_size);
++ handle->data = NULL;
++ handle->size = handle->mapping_size = 0;
++ return 0;
++}
++
++/* Make the memory map bigger by MMAP_INCREASE_SIZE. Return 0 on success */
++int
++tor_mremap_inc(tor_mmap_t *handle) {
++ /* Create a new anonymous mmap if we don't have one yet. */
++ if (handle->data==NULL)
++ return tor_mmap_anon(handle);
++
++ if (handle->size==handle->mapping_size) {
++ char *new_map;
++ int new_mapsize;
++ int page_size = getpagesize();
++
++ /* Add the increment and round up to the page boundary */
++ new_mapsize = handle->mapping_size + MMAP_INCREASE_SIZE + (MMAP_INCREASE_SIZE%page_size ? page_size-MMAP_INCREASE_SIZE%page_size : 0);
++
++ if (new_mapsize >= SIZE_T_CEILING) {
++ errno = ERANGE;
++ return -1;
++ }
++
++ /* Increase the map size */
++ if (MAP_FAILED == (new_map = mremap((void *)handle->data, handle->mapping_size, new_mapsize, MREMAP_MAYMOVE)))
++ return -1;
++
++ handle->data = new_map;
++ handle->mapping_size = new_mapsize;
++ }
++ return 0;
++}
++
++/** Read contents of <b>string</b> into a an
++ * anonymous map; return 0 on success. New data will be appended
++ * to existing data in the map. The map size will be increased if
++ * it's too small */
++int
++load_string_into_mmap(tor_mmap_t *handle, const char *string, const size_t length)
++{
++ char *pos=(char*)string;
++ ssize_t num;
++
++ tor_assert(string);
++
++ /* While there's enough data to fill up the map, fill it and increase the map size */
++ do {
++ /* When the current map is full, increase its size */
++ if (handle->size==handle->mapping_size && tor_mremap_inc(handle)) {
++ int save_errno = errno;
++ log_err(LD_FS,"Could not increase the anonymous memory mmaping while loading string");
++ errno = save_errno;
++ return -1;
++ }
++
++ /* Find out how much data we can handle in this iternation copy */
++ num = (length > handle->mapping_size-handle->size) ?
++ handle->mapping_size - handle->size : length;
++ memcpy((void *)handle->data+handle->size, pos, num);
++
++ /* advance the pointer */
++ pos+=num;
++ handle->size+=num;
++
++ /* Repeat if there was enough data and the map is full */
++ } while (handle->mapping_size==handle->size);
++
++ return 0;
++}
++
++/** Read contents of <b>filename</b> into a an
++ * anonymous map; return 0 on success. New data will be appended
++ * to existing data in the map. The map size will be increased if
++ * it's too small */
++int
++load_file_into_mmap(tor_mmap_t *handle, const char *filename)
++{
++ int fd; /* router file */
++ ssize_t r;
++
++ tor_assert(filename);
++
++ fd = tor_open_cloexec(filename,O_RDONLY|O_TEXT,0);
++ if (fd<0) {
++ int save_errno = errno;
++ log_info(LD_FS,"Could not open \"%s\": %s",filename,
++ strerror(errno));
++ errno = save_errno;
++ return -1;
++ }
++
++ /* While there's enough data to fill up the map, fill it and increase the map size */
++ do {
++ /* When the current map is full, increase its size */
++ if (handle->size==handle->mapping_size && tor_mremap_inc(handle)) {
++ int save_errno = errno;
++ log_err(LD_FS,"Could not increase the anonymous memory mmaping while reading from file \"%s\"", filename);
++ close(fd);
++ errno = save_errno;
++ return -1;
++ }
++
++ /* Fill the map with data */
++ r = read(fd,(void *)handle->data+handle->size,handle->mapping_size-handle->size);
++ if (r<0) {
++ int save_errno = errno;
++ log_warn(LD_FS,"Error reading from file \"%s\": %s", filename,
++ strerror(errno));
++ close(fd);
++ errno = save_errno;
++ return -1;
++ }
++ handle->size+=r;
++
++ /* Repeat if we haven't reached the end of the file */
++ } while (r);
++
++ close(fd);
++ return 0;
++}
+diff -uNr tor-0.2.3.19-rc/src/common/util.h tor-0.2.3.19-rc-anon_mmap//src/common/util.h
+--- tor-0.2.3.19-rc/src/common/util.h 2012-07-06 16:32:28.000000000 -0400
++++ tor-0.2.3.19-rc-anon_mmap//src/common/util.h 2012-07-09 11:39:11.995224656 -0400
+@@ -366,6 +366,12 @@
+ struct smartlist_t *tor_listdir(const char *dirname);
+ int path_is_relative(const char *filename);
+
++int tor_mmap_anon(tor_mmap_t *handle);
++int tor_mremap_inc(tor_mmap_t *handle);
++int load_string_into_mmap(tor_mmap_t *handle, const char *string, const size_t length);
++int load_file_into_mmap(tor_mmap_t *handle, const char *filename);
++int tor_munmap(tor_mmap_t *handle);
++
+ /* Process helpers */
+ void start_daemon(void);
+ void finish_daemon(const char *desired_cwd);
+diff -uNr tor-0.2.3.19-rc/src/or/control.c tor-0.2.3.19-rc-anon_mmap//src/or/control.c
+--- tor-0.2.3.19-rc/src/or/control.c 2012-07-06 16:32:28.000000000 -0400
++++ tor-0.2.3.19-rc-anon_mmap//src/or/control.c 2012-07-09 11:39:12.003224655 -0400
+@@ -34,6 +34,7 @@
+ #include "router.h"
+ #include "routerlist.h"
+ #include "routerparse.h"
++#include "microdesc.h"
+
+ #ifndef _WIN32
+ #include <pwd.h>
+@@ -1666,8 +1667,8 @@
+ const microdesc_t *md = NULL;
+ if (node) md = node->md;
+ if (md) {
+- tor_assert(md->body);
+- *answer = tor_strndup(md->body, md->bodylen);
++ tor_assert(microdesc_get_body(md));
++ *answer = tor_strndup(microdesc_get_body(md), md->bodylen);
+ }
+ } else if (!strcmpstart(question, "md/name/")) {
+ /* XXX023 Setting 'warn_if_unnamed' here is a bit silly -- the
+@@ -1677,8 +1678,8 @@
+ const microdesc_t *md = NULL;
+ if (node) md = node->md;
+ if (md) {
+- tor_assert(md->body);
+- *answer = tor_strndup(md->body, md->bodylen);
++ tor_assert(microdesc_get_body(md));
++ *answer = tor_strndup(microdesc_get_body(md), md->bodylen);
+ }
+ } else if (!strcmpstart(question, "desc-annotations/id/")) {
+ ri = router_get_by_hexdigest(question+
+diff -uNr tor-0.2.3.19-rc/src/or/microdesc.c tor-0.2.3.19-rc-anon_mmap//src/or/microdesc.c
+--- tor-0.2.3.19-rc/src/or/microdesc.c 2012-07-06 16:31:36.000000000 -0400
++++ tor-0.2.3.19-rc-anon_mmap//src/or/microdesc.c 2012-07-09 11:39:12.011224656 -0400
+@@ -65,6 +65,16 @@
+ _microdesc_hash, _microdesc_eq, 0.6,
+ malloc, realloc, free);
+
++/** Return the pointer to the body of <b>md</b>. */
++char *
++microdesc_get_body(const microdesc_t *md)
++{
++ return md->saved_location==SAVED_IN_CACHE?
++ (void*)get_microdesc_cache()->cache_content->data
++ + md->cache_offset
++ : md->body;
++}
++
+ /** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations.
+ * On success, return the total number of bytes written, and set
+ * *<b>annotation_len_out</b> to the number of bytes written as
+@@ -92,8 +102,7 @@
+ *annotation_len_out = 0;
+ }
+
+- md->off = (off_t) ftell(f);
+- written = fwrite(md->body, 1, md->bodylen, f);
++ written = fwrite(microdesc_get_body(md), 1, md->bodylen, f);
+ if (written != md->bodylen) {
+ log_warn(LD_DIR,
+ "Couldn't dump microdescriptor (wrote %lu out of %lu): %s",
+@@ -115,6 +124,9 @@
+ {
+ if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) {
+ microdesc_cache_t *cache = tor_malloc_zero(sizeof(microdesc_cache_t));
++ cache->cache_content = tor_malloc_zero(sizeof(tor_mmap_t));
++ cache->cache_content->data = NULL;
++ cache->cache_content->size = cache->cache_content->mapping_size = 0;
+ HT_INIT(microdesc_map, &cache->map);
+ cache->cache_fname = get_datadir_fname("cached-microdescs");
+ cache->journal_fname = get_datadir_fname("cached-microdescs.new");
+@@ -215,6 +227,7 @@
+ added = smartlist_new();
+ SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
+ microdesc_t *md2;
++ md->saved_location = where;
+ md2 = HT_FIND(microdesc_map, &cache->map, md);
+ if (md2) {
+ /* We already had this one. */
+@@ -229,6 +242,7 @@
+ /* Okay, it's a new one. */
+ if (f) {
+ size_t annotation_len;
++ /* Write it to the file */
+ size = dump_microdescriptor(f, md, &annotation_len);
+ if (size < 0) {
+ /* we already warned in dump_microdescriptor; */
+@@ -236,10 +250,20 @@
+ smartlist_clear(added);
+ return added;
+ }
+- md->saved_location = SAVED_IN_JOURNAL;
++ md->off = cache->journal_len + annotation_len;
++ /* Append in the anonymous mmap of microdescriptors */
++ size_t new_offset = cache->cache_content->size;
++ if (load_string_into_mmap(cache->cache_content,
++ md->body, md->bodylen))
++ /* Loading into mmap failed, so leave it as a heap chunk */
++ md->saved_location = SAVED_IN_JOURNAL;
++ else {
++ /* Body is now in mmap */
++ tor_free(md->body);
++ md->cache_offset = new_offset;
++ md->saved_location = SAVED_IN_CACHE;
++ }
+ cache->journal_len += size;
+- } else {
+- md->saved_location = where;
+ }
+
+ md->no_save = no_save;
+@@ -278,9 +302,8 @@
+ microdesc_free(md);
+ }
+ HT_CLEAR(microdesc_map, &cache->map);
+- if (cache->cache_content) {
+- tor_munmap_file(cache->cache_content);
+- cache->cache_content = NULL;
++ if (cache->cache_content->data) {
++ tor_munmap(cache->cache_content);
+ }
+ cache->total_len_seen = 0;
+ cache->n_seen = 0;
+@@ -292,36 +315,25 @@
+ int
+ microdesc_cache_reload(microdesc_cache_t *cache)
+ {
+- struct stat st;
+- char *journal_content;
+ smartlist_t *added;
+- tor_mmap_t *mm;
+ int total = 0;
+
+ microdesc_cache_clear(cache);
+
+- mm = cache->cache_content = tor_mmap_file(cache->cache_fname);
+- if (mm) {
+- added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size,
+- SAVED_IN_CACHE, 0, -1, NULL);
+- if (added) {
+- total += smartlist_len(added);
+- smartlist_free(added);
+- }
+- }
++ load_file_into_mmap(cache->cache_content, cache->cache_fname);
++ size_t cache_size = cache->cache_content->size;
+
+- journal_content = read_file_to_str(cache->journal_fname,
+- RFTS_IGNORE_MISSING, &st);
+- if (journal_content) {
+- cache->journal_len = (size_t) st.st_size;
+- added = microdescs_add_to_cache(cache, journal_content,
+- journal_content+st.st_size,
+- SAVED_IN_JOURNAL, 0, -1, NULL);
++ load_file_into_mmap(cache->cache_content, cache->journal_fname);
++ cache->journal_len = cache->cache_content->size - cache_size;
++
++ if (cache->cache_content->size) {
++ added = microdescs_add_to_cache(cache, cache->cache_content->data,
++ cache->cache_content->data+cache->cache_content->size,
++ SAVED_IN_CACHE, 0, -1, NULL);
+ if (added) {
+ total += smartlist_len(added);
+ smartlist_free(added);
+ }
+- tor_free(journal_content);
+ }
+ log_notice(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.",
+ total);
+@@ -427,7 +439,6 @@
+ log_info(LD_DIR, "Rebuilding the microdescriptor cache...");
+
+ orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0);
+- orig_size += (int)cache->journal_len;
+
+ f = start_writing_to_stdio_file(cache->cache_fname,
+ OPEN_FLAGS_REPLACE|O_BINARY,
+@@ -459,14 +470,12 @@
+ smartlist_add(wrote, md);
+ }
+
+- if (cache->cache_content)
+- tor_munmap_file(cache->cache_content);
++ if (cache->cache_content->data)
++ tor_munmap(cache->cache_content);
+
+ finish_writing_to_file(open_file); /*XXX Check me.*/
+
+- cache->cache_content = tor_mmap_file(cache->cache_fname);
+-
+- if (!cache->cache_content && smartlist_len(wrote)) {
++ if (load_file_into_mmap(cache->cache_content, cache->cache_fname) && smartlist_len(wrote)) {
+ log_err(LD_DIR, "Couldn't map file that we just wrote to %s!",
+ cache->cache_fname);
+ smartlist_free(wrote);
+@@ -474,21 +483,21 @@
+ }
+ SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) {
+ tor_assert(md->saved_location == SAVED_IN_CACHE);
+- md->body = (char*)cache->cache_content->data + md->off;
++ md->cache_offset = md->off;
+ if (PREDICT_UNLIKELY(
+- md->bodylen < 9 || fast_memneq(md->body, "onion-key", 9) != 0)) {
++ md->bodylen < 9 || fast_memneq(microdesc_get_body(md), "onion-key", 9) != 0)) {
+ /* XXXX once bug 2022 is solved, we can kill this block and turn it
+ * into just the tor_assert(!memcmp) */
+ off_t avail = cache->cache_content->size - md->off;
+ char *bad_str;
+ tor_assert(avail >= 0);
+- bad_str = tor_strndup(md->body, MIN(128, (size_t)avail));
++ bad_str = tor_strndup(microdesc_get_body(md), MIN(128, (size_t)avail));
+ log_err(LD_BUG, "After rebuilding microdesc cache, offsets seem wrong. "
+ " At offset %d, I expected to find a microdescriptor starting "
+ " with \"onion-key\". Instead I got %s.",
+ (int)md->off, escaped(bad_str));
+ tor_free(bad_str);
+- tor_assert(fast_memeq(md->body, "onion-key", 9));
++ tor_assert(fast_memeq(microdesc_get_body(md), "onion-key", 9));
+ }
+ } SMARTLIST_FOREACH_END(md);
+
+@@ -498,7 +507,7 @@
+ cache->journal_len = 0;
+ cache->bytes_dropped = 0;
+
+- new_size = cache->cache_content ? (int)cache->cache_content->size : 0;
++ new_size = (int)cache->cache_content->size;
+ log_info(LD_DIR, "Done rebuilding microdesc cache. "
+ "Saved %d bytes; %d still used.",
+ orig_size-new_size, new_size);
+@@ -594,6 +603,7 @@
+ microdesc_cache_clear(the_microdesc_cache);
+ tor_free(the_microdesc_cache->cache_fname);
+ tor_free(the_microdesc_cache->journal_fname);
++ tor_free(the_microdesc_cache->cache_content);
+ tor_free(the_microdesc_cache);
+ }
+ }
+diff -uNr tor-0.2.3.19-rc/src/or/microdesc.h tor-0.2.3.19-rc-anon_mmap//src/or/microdesc.h
+--- tor-0.2.3.19-rc/src/or/microdesc.h 2012-07-06 16:31:36.000000000 -0400
++++ tor-0.2.3.19-rc-anon_mmap//src/or/microdesc.h 2012-07-09 11:39:12.011224656 -0400
+@@ -14,6 +14,8 @@
+
+ microdesc_cache_t *get_microdesc_cache(void);
+
++char* microdesc_get_body(const microdesc_t *md);
++
+ void microdesc_check_counts(void);
+
+ smartlist_t *microdescs_add_to_cache(microdesc_cache_t *cache,
+diff -uNr tor-0.2.3.19-rc/src/or/or.h tor-0.2.3.19-rc-anon_mmap//src/or/or.h
+--- tor-0.2.3.19-rc/src/or/or.h 2012-07-06 16:31:36.000000000 -0400
++++ tor-0.2.3.19-rc-anon_mmap//src/or/or.h 2012-07-09 11:39:12.015224656 -0400
+@@ -1633,16 +1633,14 @@
+ /** The descriptor isn't stored on disk at all: the copy in memory is
+ * canonical; the saved_offset field is meaningless. */
+ SAVED_NOWHERE=0,
+- /** The descriptor is stored in the cached_routers file: the
++ /** The descriptor is stored in the cached_routers file or in the journal.
++ * In both cases the body is kept in the anonymous mmap: the
+ * signed_descriptor_body is meaningless; the signed_descriptor_len and
+- * saved_offset are used to index into the mmaped cache file. */
++ * saved_offset are used to index into the memory map. */
+ SAVED_IN_CACHE,
+- /** The descriptor is stored in the cached_routers.new file: the
++ /** The descriptor is stored in the cached_routers.new file but we've
++ * failed to get space in the anonymous mmap (memory error). The
+ * signed_descriptor_body and saved_offset fields are both set. */
+- /* FFFF (We could also mmap the file and grow the mmap as needed, or
+- * lazy-load the descriptor text by using seek and read. We don't, for
+- * now.)
+- */
+ SAVED_IN_JOURNAL
+ } saved_location_t;
+
+@@ -1696,8 +1694,8 @@
+ download_status_t ei_dl_status;
+ /** Where is the descriptor saved? */
+ saved_location_t saved_location;
+- /** If saved_location is SAVED_IN_CACHE or SAVED_IN_JOURNAL, the offset of
+- * this descriptor in the corresponding file. */
++ /** The position of the body in anonymous memory map dedicated to holding
++ * descriptors that are saved in files. */
+ off_t saved_offset;
+ /** What position is this descriptor within routerlist->routers or
+ * routerlist->old_routers? -1 for none. */
+@@ -1949,9 +1947,13 @@
+ unsigned int held_by_nodes;
+
+ /** If saved_location == SAVED_IN_CACHE, this field holds the offset of the
+- * microdescriptor in the cache. */
++ * microdescriptor in the file that holds it cache. */
+ off_t off;
+
++ /** If saved_location == SAVED_IN_CACHE, this field holds the offset to the
++ * microdescriptor body in the anonymous memory map */
++ size_t cache_offset;
++
+ /* The string containing the microdesc. */
+
+ /** A pointer to the encoded body of the microdescriptor. If the
+diff -uNr tor-0.2.3.19-rc/src/or/routerlist.c tor-0.2.3.19-rc-anon_mmap//src/or/routerlist.c
+--- tor-0.2.3.19-rc/src/or/routerlist.c 2012-07-06 16:31:36.000000000 -0400
++++ tor-0.2.3.19-rc-anon_mmap//src/or/routerlist.c 2012-07-09 11:39:12.019224656 -0400
+@@ -627,8 +627,9 @@
+ }
+
+ /** Add the signed_descriptor_t in <b>desc</b> to the router
+- * journal; change its saved_location to SAVED_IN_JOURNAL and set its
+- * offset appropriately. */
++ * journal; change its saved_location to SAVED_IN_CACHE and set its
++ * offset appropriately. The location is set to SAVED_IN_JOURNAL
++ * only if we can't move the body to the cache memory map */
+ static int
+ signed_desc_append_to_journal(signed_descriptor_t *desc,
+ desc_store_t *store)
+@@ -642,10 +643,18 @@
+ tor_free(fname);
+ return -1;
+ }
+- desc->saved_location = SAVED_IN_JOURNAL;
+ tor_free(fname);
+
+- desc->saved_offset = store->journal_len;
++ size_t mmap_offset = store->mmap->size;
++ /* If we can't move the body into the map, leave it where it is */
++ if (load_string_into_mmap(store->mmap, desc->signed_descriptor_body,
++ desc->signed_descriptor_len + desc->annotations_len))
++ desc->saved_location = SAVED_IN_JOURNAL;
++ else {
++ tor_free(desc->signed_descriptor_body);
++ desc->saved_offset = mmap_offset;
++ desc->saved_location = SAVED_IN_CACHE;
++ }
+ store->journal_len += len;
+
+ return 0;
+@@ -674,7 +683,7 @@
+ router_rebuild_store(int flags, desc_store_t *store)
+ {
+ smartlist_t *chunk_list = NULL;
+- char *fname = NULL, *fname_tmp = NULL;
++ char *fname = NULL;
+ int r = -1;
+ off_t offset = 0;
+ smartlist_t *signed_descriptors = NULL;
+@@ -705,7 +714,6 @@
+ log_info(LD_DIR, "Rebuilding %s cache", store->description);
+
+ fname = get_datadir_fname(store->fname_base);
+- fname_tmp = get_datadir_fname_suffix(store->fname_base, ".tmp");
+
+ chunk_list = smartlist_new();
+
+@@ -750,25 +758,19 @@
+ smartlist_add(chunk_list, c);
+ });
+
+- if (write_chunks_to_file(fname_tmp, chunk_list, 1)<0) {
++ if (write_chunks_to_file(fname, chunk_list, 1)<0) {
+ log_warn(LD_FS, "Error writing router store to disk.");
+ goto done;
+ }
+
+ /* Our mmap is now invalid. */
+ if (store->mmap) {
+- tor_munmap_file(store->mmap);
+- store->mmap = NULL;
+- }
+-
+- if (replace_file(fname_tmp, fname)<0) {
+- log_warn(LD_FS, "Error replacing old router store: %s", strerror(errno));
+- goto done;
++ tor_munmap(store->mmap);
+ }
+
+ errno = 0;
+- store->mmap = tor_mmap_file(fname);
+- if (! store->mmap) {
++ load_file_into_mmap(store->mmap, fname);
++ if (!store->mmap) {
+ if (errno == ERANGE) {
+ /* empty store.*/
+ if (total_expected_len) {
+@@ -811,7 +813,6 @@
+ done:
+ smartlist_free(signed_descriptors);
+ tor_free(fname);
+- tor_free(fname_tmp);
+ if (chunk_list) {
+ SMARTLIST_FOREACH(chunk_list, sized_chunk_t *, c, tor_free(c));
+ smartlist_free(chunk_list);
+@@ -826,10 +827,11 @@
+ static int
+ router_reload_router_list_impl(desc_store_t *store)
+ {
+- char *fname = NULL, *altname = NULL, *contents = NULL;
+- struct stat st;
++ char *fname = NULL, *altname = NULL;
+ int read_from_old_location = 0;
+ int extrainfo = (store->type == EXTRAINFO_STORE);
++ int r;
++ size_t cache_size;
+ time_t now = time(NULL);
+ store->journal_len = store->store_len = 0;
+
+@@ -837,60 +839,56 @@
+ if (store->fname_alt_base)
+ altname = get_datadir_fname(store->fname_alt_base);
+
+- if (store->mmap) /* get rid of it first */
+- tor_munmap_file(store->mmap);
+- store->mmap = NULL;
+-
+- store->mmap = tor_mmap_file(fname);
+- if (!store->mmap && altname && file_status(altname) == FN_FILE) {
+- read_from_old_location = 1;
++ if (store->mmap->size) /* get rid of it first */
++ tor_munmap(store->mmap);
++
++ /* Load the cache file into an anonymous map first */
++ if (load_file_into_mmap(store->mmap, fname) &&
++ altname && file_status(altname) == FN_FILE) {
+ log_notice(LD_DIR, "Couldn't read %s; trying to load routers from old "
+ "location %s.", fname, altname);
+- if ((store->mmap = tor_mmap_file(altname)))
++ if (!(load_file_into_mmap(store->mmap, altname)))
+ read_from_old_location = 1;
+ }
+ if (altname && !read_from_old_location) {
+ remove_file_if_very_old(altname, now);
+ }
+- if (store->mmap) {
+- store->store_len = store->mmap->size;
+- if (extrainfo)
+- router_load_extrainfo_from_string(store->mmap->data,
+- store->mmap->data+store->mmap->size,
+- SAVED_IN_CACHE, NULL, 0);
+- else
+- router_load_routers_from_string(store->mmap->data,
+- store->mmap->data+store->mmap->size,
+- SAVED_IN_CACHE, NULL, 0, NULL);
+- }
+
++ /* Remeber the size of the current data, so we can determine
++ * whether we've read anything from the journal */
++ cache_size = store->mmap->size;
+ tor_free(fname);
+ fname = get_datadir_fname_suffix(store->fname_base, ".new");
+- if (file_status(fname) == FN_FILE)
+- contents = read_file_to_str(fname, RFTS_BIN|RFTS_IGNORE_MISSING, &st);
++ r = 0;
++ if ((file_status(fname) == FN_FILE) && (r = load_file_into_mmap(store->mmap, fname)))
++ log_warn(LD_DIR, "Couldn't read journal %s", fname);
+ if (read_from_old_location) {
+ tor_free(altname);
+ altname = get_datadir_fname_suffix(store->fname_alt_base, ".new");
+- if (!contents)
+- contents = read_file_to_str(altname, RFTS_BIN|RFTS_IGNORE_MISSING, &st);
++ if (r)
++ load_file_into_mmap(store->mmap, altname);
+ else
+ remove_file_if_very_old(altname, now);
+ }
+- if (contents) {
++
++ /* Load router data from the map */
++ if (store->mmap->size) {
++ store->store_len = store->mmap->size;
+ if (extrainfo)
+- router_load_extrainfo_from_string(contents, NULL,SAVED_IN_JOURNAL,
+- NULL, 0);
++ router_load_extrainfo_from_string(store->mmap->data,
++ store->mmap->data+store->mmap->size,
++ SAVED_IN_CACHE, NULL, 0);
+ else
+- router_load_routers_from_string(contents, NULL, SAVED_IN_JOURNAL,
+- NULL, 0, NULL);
+- store->journal_len = (size_t) st.st_size;
+- tor_free(contents);
++ router_load_routers_from_string(store->mmap->data,
++ store->mmap->data+store->mmap->size,
++ SAVED_IN_CACHE, NULL, 0, NULL);
+ }
+
+ tor_free(fname);
+ tor_free(altname);
+
+- if (store->journal_len || read_from_old_location) {
++ /* rebuild if journal was not empty or we used old locations */
++ if ((store->mmap->size-cache_size>0) || read_from_old_location) {
+ /* Always clear the journal on startup.*/
+ router_rebuild_store(RRS_FORCE, store);
+ } else if (!extrainfo) {
+@@ -2691,6 +2689,13 @@
+
+ routerlist->desc_store.description = "router descriptors";
+ routerlist->extrainfo_store.description = "extra-info documents";
++
++ routerlist->desc_store.mmap = tor_malloc_zero(sizeof(tor_mmap_t));
++ routerlist->desc_store.mmap->data = NULL;
++ routerlist->desc_store.mmap->size = routerlist->desc_store.mmap->mapping_size = 0;
++ routerlist->extrainfo_store.mmap = tor_malloc_zero(sizeof(tor_mmap_t));
++ routerlist->extrainfo_store.mmap->data = NULL;
++ routerlist->extrainfo_store.mmap->size = routerlist->extrainfo_store.mmap->mapping_size = 0;
+ }
+ return routerlist;
+ }
+@@ -2787,10 +2792,14 @@
+ signed_descriptor_free(sd));
+ smartlist_free(rl->routers);
+ smartlist_free(rl->old_routers);
+- if (routerlist->desc_store.mmap)
+- tor_munmap_file(routerlist->desc_store.mmap);
+- if (routerlist->extrainfo_store.mmap)
+- tor_munmap_file(routerlist->extrainfo_store.mmap);
++ if (routerlist->desc_store.mmap) {
++ tor_munmap(routerlist->desc_store.mmap);
++ tor_free(routerlist->desc_store.mmap);
++ }
++ if (routerlist->extrainfo_store.mmap) {
++ tor_munmap(routerlist->extrainfo_store.mmap);
++ free(routerlist->extrainfo_store.mmap);
++ }
+ tor_free(rl);
+
+ router_dir_info_changed();
+diff -uNr tor-0.2.3.19-rc/src/or/routerparse.c tor-0.2.3.19-rc-anon_mmap//src/or/routerparse.c
+--- tor-0.2.3.19-rc/src/or/routerparse.c 2012-07-06 16:31:36.000000000 -0400
++++ tor-0.2.3.19-rc-anon_mmap//src/or/routerparse.c 2012-07-09 11:39:12.023224656 -0400
+@@ -4387,18 +4387,19 @@
+ }
+
+ md = tor_malloc_zero(sizeof(microdesc_t));
+- {
+- const char *cp = tor_memstr(s, start_of_next_microdesc-s,
++ const char *cp = tor_memstr(s, start_of_next_microdesc-s,
+ "onion-key");
+- tor_assert(cp);
++ tor_assert(cp);
+
+- md->bodylen = start_of_next_microdesc - cp;
+- if (copy_body)
+- md->body = tor_strndup(cp, md->bodylen);
+- else
+- md->body = (char*)cp;
+- md->off = cp - start;
++ md->bodylen = start_of_next_microdesc - cp;
++ if (copy_body)
++ md->body = tor_strndup(cp, md->bodylen);
++ else {
++ /* we're parsing the cache */
++ md->body = NULL;
++ md->cache_offset = cp - start;
+ }
++ md->off = cp - start;
+
+ if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) {
+ if (parse_iso_time(tok->args[0], &md->last_listed)) {
+@@ -4433,7 +4434,7 @@
+ md->exit_policy = parse_short_policy(tok->args[0]);
+ }
+
+- crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256);
++ crypto_digest256(md->digest, cp, md->bodylen, DIGEST_SHA256);
+
+ smartlist_add(result, md);
+
+diff -uNr tor-0.2.3.19-rc/src/test/test_microdesc.c tor-0.2.3.19-rc-anon_mmap//src/test/test_microdesc.c
+--- tor-0.2.3.19-rc/src/test/test_microdesc.c 2012-07-06 16:31:36.000000000 -0400
++++ tor-0.2.3.19-rc-anon_mmap//src/test/test_microdesc.c 2012-07-09 11:39:12.023224656 -0400
+@@ -55,6 +55,7 @@
+ time_t time1, time2, time3;
+ char *fn = NULL, *s = NULL;
+ (void)data;
++ tor_mmap_t *map;
+
+ options = get_options_mutable();
+ tt_assert(options);
+@@ -134,25 +135,25 @@
+ tt_int_op(md2->last_listed, ==, time2);
+ tt_int_op(md3->last_listed, ==, time3);
+
+- tt_int_op(md1->saved_location, ==, SAVED_IN_JOURNAL);
+- tt_int_op(md2->saved_location, ==, SAVED_IN_JOURNAL);
+- tt_int_op(md3->saved_location, ==, SAVED_IN_JOURNAL);
++ tt_int_op(md1->saved_location, ==, SAVED_IN_CACHE);
++ tt_int_op(md2->saved_location, ==, SAVED_IN_CACHE);
++ tt_int_op(md3->saved_location, ==, SAVED_IN_CACHE);
+
+ tt_int_op(md1->bodylen, ==, strlen(test_md1));
+ tt_int_op(md2->bodylen, ==, strlen(test_md2));
+ tt_int_op(md3->bodylen, ==, strlen(test_md3_noannotation));
+- test_mem_op(md1->body, ==, test_md1, strlen(test_md1));
+- test_mem_op(md2->body, ==, test_md2, strlen(test_md2));
+- test_mem_op(md3->body, ==, test_md3_noannotation,
++ test_mem_op(microdesc_get_body(md1), ==, test_md1, strlen(test_md1));
++ test_mem_op(microdesc_get_body(md2), ==, test_md2, strlen(test_md2));
++ test_mem_op(microdesc_get_body(md3), ==, test_md3_noannotation,
+ strlen(test_md3_noannotation));
+
+ tor_asprintf(&fn, "%s"PATH_SEPARATOR"cached-microdescs.new",
+ options->DataDirectory);
+ s = read_file_to_str(fn, RFTS_BIN, NULL);
+ tt_assert(s);
+- test_mem_op(md1->body, ==, s + md1->off, md1->bodylen);
+- test_mem_op(md2->body, ==, s + md2->off, md2->bodylen);
+- test_mem_op(md3->body, ==, s + md3->off, md3->bodylen);
++ test_mem_op(microdesc_get_body(md1), ==, s + md1->off, md1->bodylen);
++ test_mem_op(microdesc_get_body(md2), ==, s + md2->off, md2->bodylen);
++ test_mem_op(microdesc_get_body(md3), ==, s + md3->off, md3->bodylen);
+
+ tt_ptr_op(md1->family, ==, NULL);
+ tt_ptr_op(md3->family, !=, NULL);
+@@ -175,11 +176,15 @@
+
+ /* read the cache. */
+ tor_asprintf(&fn, "%s"PATH_SEPARATOR"cached-microdescs",
+- options->DataDirectory);
+- s = read_file_to_str(fn, RFTS_BIN, NULL);
+- test_mem_op(md1->body, ==, s + md1->off, strlen(test_md1));
+- test_mem_op(md2->body, ==, s + md2->off, strlen(test_md2));
+- test_mem_op(md3->body, ==, s + md3->off, strlen(test_md3_noannotation));
++ options->DataDirectory);
++ map = tor_malloc(sizeof(tor_mmap_t));
++ map->data = NULL;
++ map->size = map->mapping_size = 0;
++ load_file_into_mmap(map, fn);
++ s = (char*)map->data;
++ test_mem_op(microdesc_get_body(md1), ==, s + md1->off, strlen(test_md1));
++ test_mem_op(microdesc_get_body(md2), ==, s + md2->off, strlen(test_md2));
++ test_mem_op(microdesc_get_body(md3), ==, s + md3->off, strlen(test_md3_noannotation));
+
+ /* Okay, now we are going to forget about the cache entirely, and reload it
+ * from the disk. */
+@@ -191,9 +196,9 @@
+ test_assert(md1);
+ test_assert(md2);
+ test_assert(md3);
+- test_mem_op(md1->body, ==, s + md1->off, strlen(test_md1));
+- test_mem_op(md2->body, ==, s + md2->off, strlen(test_md2));
+- test_mem_op(md3->body, ==, s + md3->off, strlen(test_md3_noannotation));
++ test_mem_op(microdesc_get_body(md1), ==, s + md1->off, strlen(test_md1));
++ test_mem_op(microdesc_get_body(md2), ==, s + md2->off, strlen(test_md2));
++ test_mem_op(microdesc_get_body(md3), ==, s + md3->off, strlen(test_md3_noannotation));
+
+ tt_int_op(md1->last_listed, ==, time1);
+ tt_int_op(md2->last_listed, ==, time2);
+@@ -222,7 +227,8 @@
+ if (wanted)
+ SMARTLIST_FOREACH(wanted, char *, cp, tor_free(cp));
+ smartlist_free(wanted);
+- tor_free(s);
++ tor_munmap(map);
++ tor_free(map);
+ tor_free(fn);
+ }
+
39 feeds/packages/net/tor-alpha/patches/001-persistent_keys_in_etc.patch
View
@@ -0,0 +1,39 @@
+diff -uNr tor-0.2.3.15-alpha/src/or/config.c tor-0.2.3.15-alpha-persistent_keys_in_etc/src/or/config.c
+--- tor-0.2.3.15-alpha/src/or/config.c 2012-04-29 17:07:09.000000000 -0400
++++ tor-0.2.3.15-alpha-persistent_keys_in_etc/src/or/config.c 2012-05-21 13:05:10.332574460 -0400
+@@ -1458,17 +1458,13 @@
+
+ /* If needed, generate a new TLS DH prime according to the current torrc. */
+ if (server_mode(options) && options->DynamicDHGroups) {
+- char *keydir = get_datadir_fname("keys");
+- if (check_private_dir(keydir, CPD_CREATE, options->User)) {
+- tor_free(keydir);
++ char *keydir = "/etc/tor/var";
++ if (check_private_dir(keydir, CPD_CREATE, options->User))
+ return -1;
+- }
+- tor_free(keydir);
+
+ if (!old_options || !old_options->DynamicDHGroups) {
+- char *fname = get_datadir_fname2("keys", "dynamic_dh_params");
++ char *fname = "/etc/tor/var/dynamic_dh_params";
+ crypto_set_tls_dh_prime(fname);
+- tor_free(fname);
+ }
+ } else { /* clients don't need a dynamic DH prime. */
+ crypto_set_tls_dh_prime(NULL);
+diff -uNr tor-0.2.3.15-alpha/src/or/router.c tor-0.2.3.15-alpha-persistent_keys_in_etc/src/or/router.c
+--- tor-0.2.3.15-alpha/src/or/router.c 2012-04-29 17:07:09.000000000 -0400
++++ tor-0.2.3.15-alpha-persistent_keys_in_etc/src/or/router.c 2012-05-21 13:05:10.336574459 -0400
+@@ -576,10 +576,9 @@
+ }
+
+ /* 1b. Read identity key. Make it if none is found. */
+- keydir = get_datadir_fname2("keys", "secret_id_key");
++ keydir = "/etc/tor/var/secret_id_key";
+ log_info(LD_GENERAL,"Reading/making identity key \"%s\"...",keydir);
+ prkey = init_key_from_file(keydir, 1, LOG_ERR);
+- tor_free(keydir);
+ if (!prkey) return -1;
+ set_server_identity_key(prkey);
+
37 feeds/packages/net/tor-alpha/patches/001-rebuild_when_big_dropped.patch
View
@@ -0,0 +1,37 @@
+diff -uNr tor-0.2.3.11-alpha/src/or/microdesc.c tor-0.2.3.11-alpha-rebuild_when_big_dropped/src/or/microdesc.c
+--- tor-0.2.3.11-alpha/src/or/microdesc.c 2012-01-21 22:07:29.000000000 -0500
++++ tor-0.2.3.11-alpha-rebuild_when_big_dropped/src/or/microdesc.c 2012-02-07 12:16:52.139285083 -0500
+@@ -375,16 +375,14 @@
+ static int
+ should_rebuild_md_cache(microdesc_cache_t *cache)
+ {
+- const size_t old_len =
+- cache->cache_content ? cache->cache_content->size : 0;
+ const size_t journal_len = cache->journal_len;
+ const size_t dropped = cache->bytes_dropped;
+
+ if (journal_len < 16384)
+ return 0; /* Don't bother, not enough has happened yet. */
+- if (dropped > (journal_len + old_len) / 3)
+- return 1; /* We could save 1/3 or more of the currently used space. */
+- if (journal_len > old_len / 2)
++ if (dropped > 1<<18)
++ return 1; /* We could save 256KiB or more of the currently used space. */
++ if (journal_len > 1<<19)
+ return 1; /* We should append to the regular file */
+
+ return 0;
+diff -uNr tor-0.2.3.11-alpha/src/or/routerlist.c tor-0.2.3.11-alpha-rebuild_when_big_dropped/src/or/routerlist.c
+--- tor-0.2.3.11-alpha/src/or/routerlist.c 2012-01-21 22:07:30.000000000 -0500
++++ tor-0.2.3.11-alpha-rebuild_when_big_dropped/src/or/routerlist.c 2012-02-07 12:14:55.226311557 -0500
+@@ -597,8 +597,8 @@
+ router_should_rebuild_store(desc_store_t *store)
+ {
+ if (store->store_len > (1<<16))
+- return (store->journal_len > store->store_len / 2 ||
+- store->bytes_dropped > store->store_len / 2);
++ return (store->journal_len > (1<<20) ||
++ store->bytes_dropped > (1<<18));
+ else
+ return store->journal_len > (1<<15);
+ }
30 feeds/packages/net/tor-alpha/patches/001-shorter_desc_lifetime.patch
View
@@ -0,0 +1,30 @@
+diff -uNr tor-0.2.3.15-alpha/src/or/microdesc.c tor-0.2.3.15-alpha-shorter_desc_lifetime//src/or/microdesc.c
+--- tor-0.2.3.15-alpha/src/or/microdesc.c 2012-04-29 17:07:09.000000000 -0400
++++ tor-0.2.3.15-alpha-shorter_desc_lifetime//src/or/microdesc.c 2012-05-21 12:56:41.516557041 -0400
+@@ -327,7 +327,7 @@
+
+ /** By default, we remove any microdescriptors that have gone at least this
+ * long without appearing in a current consensus. */
+-#define TOLERATE_MICRODESC_AGE (7*24*60*60)
++#define TOLERATE_MICRODESC_AGE (3*24*60*60)
+
+ /** Remove all microdescriptors from <b>cache</b> that haven't been listed for
+ * a long time. Does not rebuild the cache on disk. If <b>cutoff</b> is
+diff -uNr tor-0.2.3.15-alpha/src/or/or.h tor-0.2.3.15-alpha-shorter_desc_lifetime//src/or/or.h
+--- tor-0.2.3.15-alpha/src/or/or.h 2012-04-29 17:07:09.000000000 -0400
++++ tor-0.2.3.15-alpha-shorter_desc_lifetime//src/or/or.h 2012-05-21 12:58:50.860561469 -0400
+@@ -184,12 +184,12 @@
+
+ /** How old do we allow a router to get before removing it
+ * from the router list? In seconds. */
+-#define ROUTER_MAX_AGE (60*60*48)
++#define ROUTER_MAX_AGE (60*60*24)
+ /** How old can a router get before we (as a server) will no longer
+ * consider it live? In seconds. */
+ #define ROUTER_MAX_AGE_TO_PUBLISH (60*60*24)
+ /** How old do we let a saved descriptor get before force-removing it? */
+-#define OLD_ROUTER_DESC_MAX_AGE (60*60*24*5)
++#define OLD_ROUTER_DESC_MAX_AGE (60*60*24)
+
+ /** Possible rules for generating circuit IDs on an OR connection. */
+ typedef enum {
27 feeds/packages/net/tor-alpha/patches/001-torrc.patch
View
@@ -0,0 +1,27 @@
+--- a/src/config/torrc.sample.in
++++ b/src/config/torrc.sample.in
+@@ -45,11 +45,11 @@
+ ## Uncomment this to start the process in the background... or use
+ ## --runasdaemon 1 on the command line. This is ignored on Windows;
+ ## see the FAQ entry if you want Tor to run as an NT service.
+-#RunAsDaemon 1
++RunAsDaemon 1
+
+ ## The directory for keeping all the keys/etc. By default, we store
+ ## things in $HOME/.tor on Unix, and in Application Data\tor on Windows.
+-#DataDirectory @LOCALSTATEDIR@/lib/tor
++DataDirectory @LOCALSTATEDIR@/lib/tor
+
+ ## The port on which Tor will listen for local connections from Tor
+ ## controller applications, as documented in control-spec.txt.
+@@ -187,3 +187,9 @@
+ ## address manually to your friends, uncomment this line:
+ #PublishServerDescriptor 0
+
++User tor
++VirtualAddrNetwork 10.192.0.0/10
++TransPort 9040
++TransListenAddress 172.16.1.1
++DNSPort 9053
++DNSListenAddress 172.16.1.1
+
957 feeds/packages/net/tor-alpha/patches/002-gzipped_cache.patch
View
@@ -0,0 +1,957 @@
+diff -uNr tor-0.2.3.16-alpha-anon_mmap//src/common/gzutil.c tor-0.2.3.16-alpha-gzipped_cache//src/common/gzutil.c
+--- tor-0.2.3.16-alpha-anon_mmap//src/common/gzutil.c 1969-12-31 19:00:00.000000000 -0500
++++ tor-0.2.3.16-alpha-gzipped_cache//src/common/gzutil.c 2012-06-09 14:42:20.538428664 -0400
+@@ -0,0 +1,298 @@
++/* Copyright (c) 2003, Roger Dingledine
++ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
++ * Copyright (c) 2007-2011, The Tor Project, Inc. */
++/* See LICENSE for licensing information */
++
++/**
++ * \file gzutil.c
++ * \brief Common functions for gzipped files.
++ **/
++
++#include "orconfig.h"
++#include "util.h"
++#include "gzutil.h"
++#include "torlog.h"
++#undef log
++#include "torint.h"
++#include "container.h"
++
++#ifdef MS_WINDOWS
++#include <io.h>
++#endif
++
++#include <stdlib.h>
++#include <stdio.h>
++
++#ifdef HAVE_ERRNO_H
++#include <errno.h>
++#endif
++#ifdef HAVE_UNISTD_H
++#include <unistd.h>
++#endif
++#ifdef HAVE_SYS_STAT_H
++#include <sys/stat.h>
++#endif
++#ifdef HAVE_SYS_FCNTL_H
++#include <sys/fcntl.h>
++#endif
++#ifdef HAVE_FCNTL_H
++#include <fcntl.h>
++#endif
++#ifdef HAVE_SYS_MMAN_H
++#include <sys/mman.h>
++#endif
++
++/** Represents a file that we're writing to, with support for atomic commit:
++ * we can write into a temporary file, and either remove the file on
++ * failure, or replace the original file on success. */
++struct gzopen_file_t {
++ char *filename; /**< Name of the original file. */
++ unsigned binary:1; /**< Did we open in binary mode? */
++ int fd; /**< fd for the open file. */
++ gzFile gzf; /**< gzip wrapper for <b>fb</b>. */
++};
++
++/** Write <b>count</b> bytes from <b>buf</b> to <b>fd</b>.
++ * Return the number of bytes written, or -1
++ * on error. Only use if fd is a blocking fd. */
++ssize_t
++gzwrite_all(gzFile gzf, const char *buf, size_t count)
++{
++ size_t written = 0;
++ ssize_t result;
++ tor_assert(count < SSIZE_T_MAX);
++
++ while (written != count) {
++ result = gzwrite(gzf, buf+written, count-written);
++ if (result<0)
++ return -1;
++ written += result;
++ }
++ return (ssize_t)count;
++}
++
++/** Read from <b>fd</b> to <b>buf</b>, until we get <b>count</b> bytes
++ * or reach the end of the file.
++ * Return the number of bytes read, or -1 on error. Only use
++ * if fd is a blocking fd. */
++ssize_t
++gzread_all(gzFile gzf, char *buf, size_t count)
++{
++ size_t numread = 0;
++ ssize_t result;
++
++ if (count > SIZE_T_CEILING || count > SSIZE_T_MAX)
++ return -1;
++
++ while (numread != count) {
++ result = gzread(gzf, buf+numread, count-numread);
++ if (result<0)
++ return -1;
++ else if (result == 0)
++ break;
++ numread += result;
++ }
++ return (ssize_t)numread;
++}
++
++/** Try to start writing to the file in <b>fname</b>, passing the flags
++ * <b>open_flags</b> to the open() syscall, creating the file (if needed) with
++ * access value <b>mode</b>. We open a new temporary file in the same
++ * directory, and either replace the original or remove the temporary file
++ * when we're done.
++ *
++ * Return the gzip descriptor for the newly opened file, and store working data in
++ * *<b>data_out</b>. The caller should not close the descriptor manually:
++ * instead, call finish_writing_to_file() or abort_writing_to_file().
++ * Returns NULL on failure.
++ */
++gzFile
++gzstart_writing_to_file(const char *fname, int open_flags, int mode,
++ gzopen_file_t **data_out)
++{
++ gzopen_file_t *new_file = tor_malloc_zero(sizeof(gzopen_file_t));
++ const char *open_name;
++
++ tor_assert(fname);
++ tor_assert(data_out);
++#if (O_BINARY != 0 && O_TEXT != 0)
++ tor_assert((open_flags & (O_BINARY|O_TEXT)) != 0);
++#endif
++ new_file->fd = -1;
++ open_name = new_file->filename = tor_strdup(fname);
++ open_flags |= O_CREAT;
++ open_flags &= ~O_EXCL;
++ if (open_flags & O_BINARY)
++ new_file->binary = 1;
++
++ new_file->fd = open(open_name, open_flags, mode);
++ if (new_file->fd < 0) {
++ log_warn(LD_FS, "Couldn't open \"%s\" (%s) for writing: %s",
++ open_name, fname, strerror(errno));
++ goto err;
++ }
++
++ /* Open the gzip descriptor */
++ new_file->gzf = gzdopen (new_file->fd, (open_flags & O_APPEND)?"a":"w");
++ if (!new_file->gzf) {
++ log_warn(LD_FS,"Error opening gzipped file \"%s\": %s", open_name,
++ strerror(errno));
++ goto err;
++ }
++
++ *data_out = new_file;
++
++ return new_file->gzf;
++
++ err:
++ if (new_file->fd >= 0)
++ close(new_file->fd);
++ *data_out = NULL;
++ tor_free(new_file->filename);
++ tor_free(new_file);
++ return NULL;
++}
++
++/** Helper function: close and free the underlying file and memory in
++ * <b>file_data</b>. If we were writing into a temporary file, then delete
++ * that file (if abort_write is true) or replaces the target file with
++ * the temporary file (if abort_write is false). */
++static int
++gzfinish_writing_to_file_impl(gzopen_file_t *file_data, int abort_write)
++{
++ int r = 0;
++ tor_assert(file_data && file_data->filename);
++ if (gzclose(file_data->gzf)!=Z_OK) {
++ log_warn(LD_FS, "Error closing \"%s\": %s", file_data->filename,
++ strerror(errno));
++ abort_write = r = -1;
++ close(file_data->fd);
++ }
++
++ tor_free(file_data->filename);
++ tor_free(file_data);
++
++ return r;
++}
++
++/** Finish writing to <b>file_data</b>: close the file handle, free memory as
++ * needed, and if using a temporary file, replace the original file with
++ * the temporary file. */
++int
++gzfinish_writing_to_file(gzopen_file_t *file_data)
++{
++ return gzfinish_writing_to_file_impl(file_data, 0);
++}
++
++/** Finish writing to <b>file_data</b>: close the file handle, free memory as
++ * needed, and if using a temporary file, delete it. */
++int
++gzabort_writing_to_file(gzopen_file_t *file_data)
++{
++ return gzfinish_writing_to_file_impl(file_data, 1);
++}
++
++/** Helper: given a set of flags as passed to open(2), open the file
++ * <b>fname</b> and write all the sized_chunk_t structs in <b>chunks</b> to
++ * the file. Do so as atomically as possible e.g. by opening temp files and
++ * renaming. */
++static int
++gzwrite_chunks_to_file_impl(const char *fname, const smartlist_t *chunks,
++ int open_flags)
++{
++ gzopen_file_t *file = NULL;
++ ssize_t result;
++ gzFile gzf = gzstart_writing_to_file(fname, open_flags, 0600, &file);
++ if (gzf==NULL)
++ return -1;
++ SMARTLIST_FOREACH(chunks, sized_chunk_t *, chunk,
++ {
++ result = gzwrite_all(gzf, chunk->bytes, chunk->len);
++ if (result < 0) {
++ log_warn(LD_FS, "Error writing to \"%s\": %s", fname,
++ strerror(errno));
++ goto err;
++ }
++ tor_assert((size_t)result == chunk->len);
++ });
++
++ return gzfinish_writing_to_file(file);
++ err:
++ gzabort_writing_to_file(file);
++ return -1;
++}
++
++/** Given a smartlist of sized_chunk_t, write them atomically to a file
++ * <b>fname</b>, overwriting or creating the file as necessary. */
++int
++gzwrite_chunks_to_file(const char *fname, const smartlist_t *chunks, int bin)
++{
++ int flags = OPEN_FLAGS_REPLACE|(bin?O_BINARY:O_TEXT);
++ return gzwrite_chunks_to_file_impl(fname, chunks, flags);
++}
++
++/** As write_bytes_to_file, but if the file already exists, append the bytes
++ * * to the end of the file instead of overwriting it. */
++int
++gzappend_bytes_to_file(const char *fname, const char *str, size_t len,
++ int bin)
++{
++ int r;
++ sized_chunk_t c = { str, len };
++ smartlist_t *chunks = smartlist_new();
++ smartlist_add(chunks, &c);
++ r = gzwrite_chunks_to_file_impl(fname, chunks,
++ OPEN_FLAGS_APPEND|(bin?O_BINARY:O_TEXT));
++ smartlist_free(chunks);
++ return r;
++}
++
++/** Uncompress contents of <b>filename</b> into a an
++ * anonymous map; return 0 on success. New data will be appended
++ * to existing data in the map. The map size will be increased if
++ * it's too small */
++int
++gzload_file_into_mmap(tor_mmap_t *handle, const char *filename)
++{
++ gzFile gzf; /* router file */
++ ssize_t r;
++
++ tor_assert(filename);
++
++ if (!(gzf = gzopen(filename, "rb"))) {
++ int save_errno = errno;
++ log_info(LD_FS,"Could not open \"%s\": %s",filename,
++ strerror(errno));
++ errno = save_errno;
++ return -1;
++ }
++
++ /* While there's enough data to fill up the map, fill it and increase the map size */
++ do {
++ /* When the current map is full, increase its size */
++ if (handle->size==handle->mapping_size && tor_mremap_inc(handle)) {
++ int save_errno = errno;
++ log_err(LD_FS,"Could not increase the anonymous memory mmaping while reading from file \"%s\"", filename);
++ gzclose(gzf);
++ errno = save_errno;
++ return -1;
++ }
++
++ /* Fill the map with data */
++ r = gzread(gzf,(void*)handle->data+handle->size,handle->mapping_size-handle->size);
++ if (r<0) {
++ int save_errno = errno;
++ log_warn(LD_FS,"Error reading from file \"%s\": %s", filename,
++ strerror(errno));
++ gzclose(gzf);
++ errno = save_errno;
++ return -1;
++ }
++ handle->size+=r;
++
++ /* Repeat if we haven't reached the end of the file */
++ } while (r);
++
++ gzclose(gzf);
++ return 0;
++}
+diff -uNr tor-0.2.3.16-alpha-anon_mmap//src/common/gzutil.h tor-0.2.3.16-alpha-gzipped_cache//src/common/gzutil.h
+--- tor-0.2.3.16-alpha-anon_mmap//src/common/gzutil.h 1969-12-31 19:00:00.000000000 -0500
++++ tor-0.2.3.16-alpha-gzipped_cache//src/common/gzutil.h 2012-06-11 18:29:08.043767984 -0400
+@@ -0,0 +1,47 @@
++/* Copyright (c) 2003-2004, Roger Dingledine
++ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
++ * Copyright (c) 2007-2011, The Tor Project, Inc. */
++/* See LICENSE for licensing information */
++
++/**
++ * \file gzutil.h
++ * \brief Headers for gzutil.c
++ **/
++
++#ifndef _TOR_GZUTIL_H
++#define _TOR_GZUTIL_H
++
++#include "orconfig.h"
++#include "torint.h"
++#include "compat.h"
++#include "di_ops.h"
++#include "util.h"
++#include <stdio.h>
++#include <stdlib.h>
++
++#include <zlib.h>
++
++#ifndef O_BINARY
++#define O_BINARY 0
++#endif
++#ifndef O_TEXT
++#define O_TEXT 0
++#endif
++
++typedef struct gzopen_file_t gzopen_file_t;
++
++int gzwrite_chunks_to_file(const char *fname, const struct smartlist_t *chunks,
++ int bin);
++
++int gzload_file_into_mmap(tor_mmap_t *handle, const char *filename);
++
++ssize_t gzwrite_all(gzFile gzf, const char *buf, size_t count);
++
++gzFile gzstart_writing_to_file(const char *fname, int open_flags, int mode,
++ gzopen_file_t **data_out);
++int gzfinish_writing_to_file(gzopen_file_t *file_data);
++int gzabort_writing_to_file(gzopen_file_t *file_data);
++int gzappend_bytes_to_file(const char *fname, const char *str, size_t len,
++ int bin);
++
++#endif
+diff -uNr tor-0.2.3.16-alpha-anon_mmap//src/common/Makefile.am tor-0.2.3.16-alpha-gzipped_cache//src/common/Makefile.am
+--- tor-0.2.3.16-alpha-anon_mmap//src/common/Makefile.am 2012-06-02 01:01:17.000000000 -0400
++++ tor-0.2.3.16-alpha-gzipped_cache//src/common/Makefile.am 2012-06-09 14:42:20.538428664 -0400
+@@ -21,6 +21,7 @@
+ mempool.c \
+ procmon.c \
+ util.c \
++ gzutil.c \
+ util_codedigest.c \
+ $(libor_extra_source)
+
+@@ -51,7 +52,8 @@
+ torint.h \
+ torlog.h \
+ tortls.h \
+- util.h
++ util.h \
++ gzutil.h
+
+ common_sha1.i: $(libor_SOURCES) $(libor_crypto_a_SOURCES) $(noinst_HEADERS)
+ if test "@SHA1SUM@" != none; then \
+diff -uNr tor-0.2.3.16-alpha-anon_mmap//src/common/Makefile.in tor-0.2.3.16-alpha-gzipped_cache//src/common/Makefile.in
+--- tor-0.2.3.16-alpha-anon_mmap//src/common/Makefile.in 2012-06-05 12:09:50.000000000 -0400
++++ tor-0.2.3.16-alpha-gzipped_cache//src/common/Makefile.in 2012-06-09 14:42:20.586428664 -0400
+@@ -61,14 +61,14 @@
+ libor_a_AR = $(AR) $(ARFLAGS)
+ libor_a_LIBADD =
+ am__libor_a_SOURCES_DIST = address.c compat.c container.c di_ops.c \
+- log.c memarea.c mempool.c procmon.c util.c util_codedigest.c \
++ log.c memarea.c mempool.c procmon.c util.c gzutil.c util_codedigest.c \
+ OpenBSD_malloc_Linux.c
+ @USE_OPENBSD_MALLOC_TRUE@am__objects_1 = \
+ @USE_OPENBSD_MALLOC_TRUE@ OpenBSD_malloc_Linux.$(OBJEXT)
+ am_libor_a_OBJECTS = address.$(OBJEXT) compat.$(OBJEXT) \
+ container.$(OBJEXT) di_ops.$(OBJEXT) log.$(OBJEXT) \
+ memarea.$(OBJEXT) mempool.$(OBJEXT) procmon.$(OBJEXT) \
+- util.$(OBJEXT) util_codedigest.$(OBJEXT) $(am__objects_1)
++ util.$(OBJEXT) gzutil.$(OBJEXT) util_codedigest.$(OBJEXT) $(am__objects_1)
+ libor_a_OBJECTS = $(am_libor_a_OBJECTS)
+ DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)
+ depcomp = $(SHELL) $(top_srcdir)/depcomp
+@@ -227,6 +227,7 @@
+ mempool.c \
+ procmon.c \
+ util.c \
++ gzutil.c \
+ util_codedigest.c \
+ $(libor_extra_source)
+
+@@ -256,7 +257,8 @@
+ torint.h \
+ torlog.h \
+ tortls.h \
+- util.h
++ util.h \
++ gzutil.h
+
+ all: all-am
+
+@@ -329,6 +331,7 @@
+ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/torgzip.Po@am__quote@
+ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tortls.Po@am__quote@
+ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/util.Po@am__quote@
++@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gzutil.Po@am__quote@
+ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/util_codedigest.Po@am__quote@
+
+ .c.o:
+diff -uNr tor-0.2.3.16-alpha-anon_mmap//src/or/microdesc.c tor-0.2.3.16-alpha-gzipped_cache//src/or/microdesc.c
+--- tor-0.2.3.16-alpha-anon_mmap//src/or/microdesc.c 2012-06-09 14:32:35.158408624 -0400
++++ tor-0.2.3.16-alpha-gzipped_cache//src/or/microdesc.c 2012-06-11 18:26:25.163762408 -0400
+@@ -13,10 +13,13 @@
+ #include "router.h"
+ #include "routerlist.h"
+ #include "routerparse.h"
++#include "gzutil.h"
+
+-/** A data structure to hold a bunch of cached microdescriptors. There are
+- * two active files in the cache: a "cache file" that we mmap, and a "journal
+- * file" that we append to. Periodically, we rebuild the cache file to hold
++#include <zlib.h>
++
++/** A data structure to hold a bunch of cached microdescriptors. There is
++ * one active file in the cache: it's a main "cache file", and we also append
++ * the journal to it. Periodically, we rebuild the cache file to hold
+ * only the microdescriptors that we want to keep */
+ struct microdesc_cache_t {
+ /** Map from sha256-digest to microdesc_t for every microdesc_t in the
+@@ -25,8 +28,8 @@
+
+ /** Name of the cache file. */
+ char *cache_fname;
+- /** Name of the journal file. */
+- char *journal_fname;
++ /** Name of the old file. */
++ char *cache_faltname;
+ /** Mmap'd contents of the cache file, or NULL if there is none. */
+ tor_mmap_t *cache_content;
+ /** Number of bytes used in the journal file. */
+@@ -75,12 +78,12 @@
+ : md->body;
+ }
+
+-/** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations.
++/** Write the body of <b>md</b> into <b>gzf</b>, with appropriate annotations.
+ * On success, return the total number of bytes written, and set
+ * *<b>annotation_len_out</b> to the number of bytes written as
+ * annotations. */
+ static ssize_t
+-dump_microdescriptor(FILE *f, microdesc_t *md, size_t *annotation_len_out)
++gzdump_microdescriptor(gzFile gzf, microdesc_t *md, size_t *annotation_len_out)
+ {
+ ssize_t r = 0;
+ size_t written;
+@@ -90,24 +93,24 @@
+ char annotation[ISO_TIME_LEN+32];
+ format_iso_time(buf, md->last_listed);
+ tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf);
+- if (fputs(annotation, f) < 0) {
++ r += strlen(annotation);
++ if (gzwrite_all(gzf, annotation, r) < r) {
+ log_warn(LD_DIR,
+ "Couldn't write microdescriptor annotation: %s",
+- strerror(ferror(f)));
++ gzerror(gzf, NULL));
+ return -1;
+ }
+- r += strlen(annotation);
+ *annotation_len_out = r;
+ } else {
+ *annotation_len_out = 0;
+ }
+
+- written = fwrite(microdesc_get_body(md), 1, md->bodylen, f);
++ written = gzwrite_all(gzf, microdesc_get_body(md), md->bodylen);
+ if (written != md->bodylen) {
+ log_warn(LD_DIR,
+ "Couldn't dump microdescriptor (wrote %lu out of %lu): %s",
+ (unsigned long)written, (unsigned long)md->bodylen,
+- strerror(ferror(f)));
++ gzerror(gzf, NULL));
+ return -1;
+ }
+ r += md->bodylen;
+@@ -128,10 +131,12 @@
+ cache->cache_content->data = NULL;
+ cache->cache_content->size = cache->cache_content->mapping_size = 0;
+ HT_INIT(microdesc_map, &cache->map);
+- cache->cache_fname = get_datadir_fname("cached-microdescs");
+- cache->journal_fname = get_datadir_fname("cached-microdescs.new");
+- microdesc_cache_reload(cache);
++ cache->cache_fname = get_datadir_fname("cached-microdescs.gz");
++ cache->cache_faltname = get_datadir_fname("cached-microdescs");
++ /* update the global variable here as we may refer to it
++ * during reload/rebuild (while writing bodies to file) */
+ the_microdesc_cache = cache;
++ microdesc_cache_reload(cache);
+ }
+ return the_microdesc_cache;
+ }
+@@ -208,18 +213,18 @@
+ int no_save)
+ {
+ smartlist_t *added;
+- open_file_t *open_file = NULL;
+- FILE *f = NULL;
++ gzopen_file_t *open_file = NULL;
++ gzFile gzf = NULL;
+ // int n_added = 0;
+ ssize_t size = 0;
+
+ if (where == SAVED_NOWHERE && !no_save) {
+- f = start_writing_to_stdio_file(cache->journal_fname,
++ gzf = gzstart_writing_to_file(cache->cache_fname,
+ OPEN_FLAGS_APPEND|O_BINARY,
+ 0600, &open_file);
+- if (!f) {
++ if (!gzf) {
+ log_warn(LD_DIR, "Couldn't append to journal in %s: %s",
+- cache->journal_fname, strerror(errno));
++ cache->cache_fname, strerror(errno));
+ return NULL;
+ }
+ }
+@@ -240,13 +245,13 @@
+ }
+
+ /* Okay, it's a new one. */
+- if (f) {
++ if (gzf) {
+ size_t annotation_len;
+ /* Write it to the file */
+- size = dump_microdescriptor(f, md, &annotation_len);
++ size = gzdump_microdescriptor(gzf, md, &annotation_len);
+ if (size < 0) {
+ /* we already warned in dump_microdescriptor; */
+- abort_writing_to_file(open_file);
++ gzabort_writing_to_file(open_file);
+ smartlist_clear(added);
+ return added;
+ }
+@@ -275,8 +280,8 @@
+ cache->total_len_seen += md->bodylen;
+ } SMARTLIST_FOREACH_END(md);
+
+- if (f)
+- finish_writing_to_file(open_file); /*XXX Check me.*/
++ if (gzf)
++ gzfinish_writing_to_file(open_file); /*XXX Check me.*/
+
+ {
+ networkstatus_t *ns = networkstatus_get_latest_consensus();
+@@ -320,11 +325,22 @@
+
+ microdesc_cache_clear(cache);
+
+- load_file_into_mmap(cache->cache_content, cache->cache_fname);
+- size_t cache_size = cache->cache_content->size;
+-
+- load_file_into_mmap(cache->cache_content, cache->journal_fname);
+- cache->journal_len = cache->cache_content->size - cache_size;
++ /* Load the cache file into an anonymous map first */
++ if (gzload_file_into_mmap(cache->cache_content, cache->cache_fname) &&
++ cache->cache_faltname) {
++ log_notice(LD_DIR, "Couldn't read %s; trying to load routers from old "
++ "uncompressed location %s.",
++ cache->cache_fname, cache->cache_faltname);
++ /* Read the old cache */
++ if (file_status(cache->cache_faltname) == FN_FILE)
++ load_file_into_mmap(cache->cache_content, cache->cache_faltname);
++ /* Load the old journal */
++ char *altname;
++ if (tor_asprintf(&altname, "%s.new", cache->cache_faltname)!=-1 &&
++ file_status(altname) == FN_FILE)
++ load_file_into_mmap(cache->cache_content, altname);
++ tor_free(altname);
++ }
+
+ if (cache->cache_content->size) {
+ added = microdescs_add_to_cache(cache, cache->cache_content->data,
+@@ -334,12 +350,13 @@
+ total += smartlist_len(added);
+ smartlist_free(added);
+ }
++ /* Force rebuild because we don't know how much of the
++ * cache file had been journaled */
++ microdesc_cache_rebuild(cache, 1);
+ }
+ log_notice(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.",
+ total);
+
+- microdesc_cache_rebuild(cache, 0 /* don't force */);
+-
+ return 0;
+ }
+
+@@ -409,15 +426,15 @@
+ return 0;
+ }
+
+-/** Regenerate the main cache file for <b>cache</b>, clear the journal file,
++/** Regenerate the main cache file for <b>cache</b>
+ * and update every microdesc_t in the cache with pointers to its new
+ * location. If <b>force</b> is true, do this unconditionally. If
+ * <b>force</b> is false, do it only if we expect to save space on disk. */
+ int
+ microdesc_cache_rebuild(microdesc_cache_t *cache, int force)
+ {
+- open_file_t *open_file;
+- FILE *f;
++ gzopen_file_t *open_file;
++ gzFile gzf;
+ microdesc_t **mdp;
+ smartlist_t *wrote;
+ ssize_t size;
+@@ -440,10 +457,10 @@
+
+ orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0);
+
+- f = start_writing_to_stdio_file(cache->cache_fname,
++ gzf = gzstart_writing_to_file(cache->cache_fname,
+ OPEN_FLAGS_REPLACE|O_BINARY,
+ 0600, &open_file);
+- if (!f)
++ if (!gzf)
+ return -1;
+
+ wrote = smartlist_new();
+@@ -454,7 +471,7 @@
+ if (md->no_save)
+ continue;
+
+- size = dump_microdescriptor(f, md, &annotation_len);
++ size = gzdump_microdescriptor(gzf, md, &annotation_len);
+ if (size < 0) {
+ /* XXX handle errors from dump_microdescriptor() */
+ /* log? return -1? die? coredump the universe? */
+@@ -473,9 +490,9 @@
+ if (cache->cache_content->data)
+ tor_munmap(cache->cache_content);
+
+- finish_writing_to_file(open_file); /*XXX Check me.*/
++ gzfinish_writing_to_file(open_file); /*XXX Check me.*/
+
+- if (load_file_into_mmap(cache->cache_content, cache->cache_fname) && smartlist_len(wrote)) {
++ if (gzload_file_into_mmap(cache->cache_content, cache->cache_fname) && smartlist_len(wrote)) {
+ log_err(LD_DIR, "Couldn't map file that we just wrote to %s!",
+ cache->cache_fname);
+ smartlist_free(wrote);
+@@ -503,11 +520,10 @@
+
+ smartlist_free(wrote);
+
+- write_str_to_file(cache->journal_fname, "", 1);
+ cache->journal_len = 0;
+ cache->bytes_dropped = 0;
+
+- new_size = (int)cache->cache_content->size;
++ new_size = cache->cache_content ? (int)cache->cache_content->size : 0;
+ log_info(LD_DIR, "Done rebuilding microdesc cache. "
+ "Saved %d bytes; %d still used.",
+ orig_size-new_size, new_size);
+@@ -602,7 +618,7 @@
+ if (the_microdesc_cache) {
+ microdesc_cache_clear(the_microdesc_cache);
+ tor_free(the_microdesc_cache->cache_fname);
+- tor_free(the_microdesc_cache->journal_fname);
++ tor_free(the_microdesc_cache->cache_faltname);
+ tor_free(the_microdesc_cache->cache_content);
+ tor_free(the_microdesc_cache);
+ }
+diff -uNr tor-0.2.3.16-alpha-anon_mmap//src/or/or.h tor-0.2.3.16-alpha-gzipped_cache//src/or/or.h
+--- tor-0.2.3.16-alpha-anon_mmap//src/or/or.h 2012-06-09 14:28:32.798400326 -0400
++++ tor-0.2.3.16-alpha-gzipped_cache//src/or/or.h 2012-06-09 14:42:20.590428665 -0400
+@@ -1633,14 +1633,16 @@
+ /** The descriptor isn't stored on disk at all: the copy in memory is
+ * canonical; the saved_offset field is meaningless. */
+ SAVED_NOWHERE=0,
+- /** The descriptor is stored in the cached_routers file or in the journal.
+- * In both cases the body is kept in the anonymous mmap: the
++ /** The descriptor is stored in the cached_routers file: the
+ * signed_descriptor_body is meaningless; the signed_descriptor_len and
+- * saved_offset are used to index into the memory map. */
++ * saved_offset are used to index into the mmaped cache file. */
+ SAVED_IN_CACHE,
+- /** The descriptor is stored in the cached_routers.new file but we've
+- * failed to get space in the anonymous mmap (memory error). The
++ /** The descriptor is stored in the cached_routers.new file: the
+ * signed_descriptor_body and saved_offset fields are both set. */
++ /* FFFF (We could also mmap the file and grow the mmap as needed, or
++ * lazy-load the descriptor text by using seek and read. We don't, for
++ * now.)
++ */
+ SAVED_IN_JOURNAL
+ } saved_location_t;
+
+@@ -1694,8 +1696,8 @@
+ download_status_t ei_dl_status;
+ /** Where is the descriptor saved? */
+ saved_location_t saved_location;
+- /** The position of the body in anonymous memory map dedicated to holding
+- * descriptors that are saved in files. */
++ /** If saved_location is SAVED_IN_CACHE or SAVED_IN_JOURNAL, the offset of
++ * this descriptor in the corresponding file. */
+ off_t saved_offset;
+ /** What position is this descriptor within routerlist->routers or
+ * routerlist->old_routers? -1 for none. */
+@@ -1947,11 +1949,11 @@
+ unsigned int held_by_nodes;
+
+ /** If saved_location == SAVED_IN_CACHE, this field holds the offset of the
+- * microdescriptor in the file that holds it cache. */
++ * microdescriptor in the cache. */
+ off_t off;
+
+ /** If saved_location == SAVED_IN_CACHE, this field holds the offset to the
+- * microdescriptor body in the anonymous memory map */
++ * microdescriptor body in memory relative to the cache. */
+ size_t cache_offset;
+
+ /* The string containing the microdesc. */
+diff -uNr tor-0.2.3.16-alpha-anon_mmap//src/or/routerlist.c tor-0.2.3.16-alpha-gzipped_cache//src/or/routerlist.c
+--- tor-0.2.3.16-alpha-anon_mmap//src/or/routerlist.c 2012-06-09 14:28:32.802400324 -0400
++++ tor-0.2.3.16-alpha-gzipped_cache//src/or/routerlist.c 2012-06-09 14:42:20.594428666 -0400
+@@ -33,6 +33,7 @@
+ #include "router.h"
+ #include "routerlist.h"
+ #include "routerparse.h"
++#include "gzutil.h"
+
+ // #define DEBUG_ROUTERLIST
+
+@@ -583,15 +584,17 @@
+
+ /* Router descriptor storage.
+ *
+- * Routerdescs are stored in a big file, named "cached-descriptors". As new
+- * routerdescs arrive, we append them to a journal file named
+- * "cached-descriptors.new".
++ * Routerdescs are stored in a big file, named "cached-descriptors.gz". As new
++ * routerdescs arrive, we append them there instead of a separate journal file.
+ *
+- * From time to time, we replace "cached-descriptors" with a new file
+- * containing only the live, non-superseded descriptors, and clear
+- * cached-routers.new.
++ * From time to time, we replace "cached-descriptors.gz" with a new file
++ * containing only the live, non-superseded descriptors.
+ *
+- * On startup, we read both files.
++ * On startup, if we haven't read anything from "cached-descriptors.gz", we try
++ * to read uncompressed data files from a previous version of Tor.
++ *
++ * "Journal" refers to the data that has been appended to the cache file
++ * since the reload/rebuild.
+ */
+
+ /** Helper: return 1 iff the router log is so big we want to rebuild the
+@@ -618,18 +621,17 @@
+ }
+
+ /** Add the signed_descriptor_t in <b>desc</b> to the router
+- * journal; change its saved_location to SAVED_IN_CACHE and set its
+- * offset appropriately. The location is set to SAVED_IN_JOURNAL
+- * only if we can't move the body to the cache memory map */
++ * journal; change its saved_location to SAVED_IN_JOURNAL and set its
++ * offset appropriately. */
+ static int
+ signed_desc_append_to_journal(signed_descriptor_t *desc,
+ desc_store_t *store)
+ {
+- char *fname = get_datadir_fname_suffix(store->fname_base, ".new");
++ char *fname = get_datadir_fname(store->fname_base);
+ const char *body = signed_descriptor_get_body_impl(desc,1);
+ size_t len = desc->signed_descriptor_len + desc->annotations_len;
+
+- if (append_bytes_to_file(fname, body, len, 1)) {
++ if (gzappend_bytes_to_file(fname, body, len, 1)) {
+ log_warn(LD_FS, "Unable to store router descriptor");
+ tor_free(fname);
+ return -1;
+@@ -749,7 +751,7 @@
+ smartlist_add(chunk_list, c);
+ });
+
+- if (write_chunks_to_file(fname, chunk_list, 1)<0) {
++ if (gzwrite_chunks_to_file(fname, chunk_list, 1)<0) {
+ log_warn(LD_FS, "Error writing router store to disk.");
+ goto done;
+ }
+@@ -760,7 +762,7 @@
+ }
+
+ errno = 0;
+- load_file_into_mmap(store->mmap, fname);
++ gzload_file_into_mmap(store->mmap, fname);
+ if (!store->mmap) {
+ if (errno == ERANGE) {
+ /* empty store.*/
+@@ -793,10 +795,6 @@
+ signed_descriptor_get_body(sd); /* reconstruct and assert */
+ });
+
+- tor_free(fname);
+- fname = get_datadir_fname_suffix(store->fname_base, ".new");
+- write_str_to_file(fname, "", 1);
+-
+ r = 0;
+ store->store_len = (size_t) offset;
+ store->journal_len = 0;
+@@ -819,10 +817,7 @@
+ router_reload_router_list_impl(desc_store_t *store)
+ {
+ char *fname = NULL, *altname = NULL;
+- int read_from_old_location = 0;
+ int extrainfo = (store->type == EXTRAINFO_STORE);
+- int r;
+- size_t cache_size;
+ time_t now = time(NULL);
+ store->journal_len = store->store_len = 0;
+
+@@ -834,32 +829,19 @@
+ tor_munmap(store->mmap);
+
+ /* Load the cache file into an anonymous map first */
+- if (load_file_into_mmap(store->mmap, fname) &&
+- altname && file_status(altname) == FN_FILE) {
++ if (gzload_file_into_mmap(store->mmap, fname) && altname) {
+ log_notice(LD_DIR, "Couldn't read %s; trying to load routers from old "
+- "location %s.", fname, altname);
+- if (!(load_file_into_mmap(store->mmap, altname)))
+- read_from_old_location = 1;
+- }
+- if (altname && !read_from_old_location) {
++ "uncompressed location %s.", fname, altname);
++ /* Read the old cache */
++ if (file_status(altname) == FN_FILE)
++ load_file_into_mmap(store->mmap, altname);
+ remove_file_if_very_old(altname, now);
+- }
+-
+- /* Remeber the size of the current data, so we can determine
+- * whether we've read anything from the journal */
+- cache_size = store->mmap->size;
+- tor_free(fname);
+- fname = get_datadir_fname_suffix(store->fname_base, ".new");
+- r = 0;
+- if ((file_status(fname) == FN_FILE) && (r = load_file_into_mmap(store->mmap, fname)))
+- log_warn(LD_DIR, "Couldn't read journal %s", fname);
+- if (read_from_old_location) {
+ tor_free(altname);
++ /* Load the old journal */
+ altname = get_datadir_fname_suffix(store->fname_alt_base, ".new");
+- if (r)
++ if (file_status(altname) == FN_FILE)
+ load_file_into_mmap(store->mmap, altname);
+- else
+- remove_file_if_very_old(altname, now);
++ remove_file_if_very_old(altname, now);
+ }
+
+ /* Load router data from the map */
+@@ -878,8 +860,9 @@
+ tor_free(fname);
+ tor_free(altname);
+
+- /* rebuild if journal was not empty or we used old locations */
+- if ((store->mmap->size-cache_size>0) || read_from_old_location) {
++ /* Rebuild if we've read any data since we don't know
++ * how much of it is a journal */
++ if (store->mmap->size) {
+ /* Always clear the journal on startup.*/
+ router_rebuild_store(RRS_FORCE, store);
+ } else if (!extrainfo) {
+@@ -2660,9 +2643,9 @@
+ routerlist->desc_by_eid_map = sdmap_new();
+ routerlist->extra_info_map = eimap_new();
+
+- routerlist->desc_store.fname_base = "cached-descriptors";
+- routerlist->desc_store.fname_alt_base = "cached-routers";
+- routerlist->extrainfo_store.fname_base = "cached-extrainfo";
++ routerlist->desc_store.fname_base = "cached-descriptors.gz";
++ routerlist->desc_store.fname_alt_base = "cached-descriptors";
++ routerlist->extrainfo_store.fname_base = "cached-extrainfo.gz";
+
+ routerlist->desc_store.type = ROUTER_STORE;
+ routerlist->extrainfo_store.type = EXTRAINFO_STORE;
+diff -uNr tor-0.2.3.16-alpha-anon_mmap//src/test/test_microdesc.c tor-0.2.3.16-alpha-gzipped_cache//src/test/test_microdesc.c
+--- tor-0.2.3.16-alpha-anon_mmap//src/test/test_microdesc.c 2012-06-09 14:28:32.806400322 -0400
++++ tor-0.2.3.16-alpha-gzipped_cache//src/test/test_microdesc.c 2012-06-09 14:42:20.594428666 -0400
+@@ -3,6 +3,7 @@
+
+ #include "orconfig.h"
+ #include "or.h"
++#include "gzutil.h"
+
+ #include "config.h"
+ #include "microdesc.h"
+@@ -147,9 +148,13 @@
+ test_mem_op(microdesc_get_body(md3), ==, test_md3_noannotation,
+ strlen(test_md3_noannotation));
+
+- tor_asprintf(&fn, "%s"PATH_SEPARATOR"cached-microdescs.new",
++ tor_asprintf(&fn, "%s"PATH_SEPARATOR"cached-microdescs.gz",
+ options->DataDirectory);
+- s = read_file_to_str(fn, RFTS_BIN, NULL);
++ map = tor_malloc(sizeof(tor_mmap_t));
++ map->data = NULL;
++ map->size = map->mapping_size = 0;
++ gzload_file_into_mmap(map, fn);
++ s = (char*)map->data;
+ tt_assert(s);
+ test_mem_op(microdesc_get_body(md1), ==, s + md1->off, md1->bodylen);
+ test_mem_op(microdesc_get_body(md2), ==, s + md2->off, md2->bodylen);
+@@ -160,6 +165,9 @@
+ tt_int_op(smartlist_len(md3->family), ==, 3);
+ tt_str_op(smartlist_get(md3->family, 0), ==, "nodeX");
+
++ tor_munmap(map);
++ free(map);
++
+ /* Now rebuild the cache! */
+ tt_int_op(microdesc_cache_rebuild(mc, 1), ==, 0);
+
+@@ -167,20 +175,11 @@
+ tt_int_op(md2->saved_location, ==, SAVED_IN_CACHE);
+ tt_int_op(md3->saved_location, ==, SAVED_IN_CACHE);
+
+- /* The journal should be empty now */
+- tor_free(s);
+- s = read_file_to_str(fn, RFTS_BIN, NULL);
+- tt_str_op(s, ==, "");
+- tor_free(s);
+- tor_free(fn);
+-
+ /* read the cache. */
+- tor_asprintf(&fn, "%s"PATH_SEPARATOR"cached-microdescs",
+- options->DataDirectory);
+ map = tor_malloc(sizeof(tor_mmap_t));
+ map->data = NULL;
+ map->size = map->mapping_size = 0;
+- load_file_into_mmap(map, fn);
++ gzload_file_into_mmap(map, fn);
+ s = (char*)map->data;
+ test_mem_op(microdesc_get_body(md1), ==, s + md1->off, strlen(test_md1));
+ test_mem_op(microdesc_get_body(md2), ==, s + md2->off, strlen(test_md2));
Please sign in to comment.
Something went wrong with that request. Please try again.