Skip to content

Commit

Permalink
Delete account and leasedb functionality that is too ambitious for now.
Browse files Browse the repository at this point in the history 
Revert all changes to storage_client.

Signed-off-by: David-Sarah Hopwood <david-sarah@jacaranda.org>
  • Loading branch information
David-Sarah Hopwood committed Oct 1, 2012
1 parent 171000d commit 754d326
Show file tree
Hide file tree
Showing 6 changed files with 36 additions and 322 deletions.
90 changes: 14 additions & 76 deletions src/allmydata/storage/account.py
View file
Expand Up @@ -32,14 +32,6 @@ def __init__(self, owner_num, pubkey_vs, server, leasedb):
self.connected = False
self.connected_since = None
self.connection = None
self.can_write = True
self.can_read = True
self.can_save = True
import random
self.account_message = {
"message": "free storage! %d" % random.randint(0,10),
"fancy": "free pony if you knew how to ask",
}
self.debug = True

def is_static(self):
Expand Down Expand Up @@ -158,91 +150,37 @@ def remote_advise_corrupt_share(self, share_type, storage_index, shnum, reason):
return self.server.client_advise_corrupt_share(
share_type, storage_index, shnum, reason)


# these are the non-RIStorageServer methods, some remote, some local

def get_account_attribute(self, name):
return self._leasedb.get_account_attribute(self.owner_num, name)

def set_account_attribute(self, name, value):
self._leasedb.set_account_attribute(self.owner_num, name, value)

def get_account_creation_time(self):
return self._leasedb.get_account_creation_time(self.owner_num)

def remote_get_status(self):
return self.get_status()

def get_status(self):
return { "write": self.can_write,
"read": self.can_read,
"save": self.can_save,
}

def remote_get_account_message(self):
return self.account_message

def set_nickname(self, nickname):
if len(nickname) > 1000:
raise ValueError("nickname too long")
self.set_account_attribute("nickname", nickname)

def get_nickname(self):
n = self.get_account_attribute("nickname")
if n:
return n
return u""

def get_id(self):
return self.pubkey_vs

def remote_get_current_usage(self):
return self.get_current_usage()

def get_current_usage(self):
return self._leasedb.get_account_usage(self.owner_num)

def get_leases(self, storage_index):
return self._leasedb.get_leases(storage_index, self.owner_num)

def connection_from(self, rx):
self.connected = True
self.connected_since = time.time()
self.connection = rx
rhost = rx.getPeer()
from twisted.internet import address
if isinstance(rhost, address.IPv4Address):
rhost_s = "%s:%d" % (rhost.host, rhost.port)
elif "LoopbackAddress" in str(rhost):
rhost_s = "loopback"
else:
rhost_s = str(rhost)
self.set_account_attribute("last_connected_from", rhost_s)
#rhost = rx.getPeer()
#from twisted.internet import address
#if isinstance(rhost, address.IPv4Address):
# rhost_s = "%s:%d" % (rhost.host, rhost.port)
#elif "LoopbackAddress" in str(rhost):
# rhost_s = "loopback"
#else:
# rhost_s = str(rhost)
#self.set_account_attribute("last_connected_from", rhost_s)
rx.notifyOnDisconnect(self._disconnected)

def _disconnected(self):
self.connected = False
self.connected_since = None
self.connection = None
self.set_account_attribute("last_seen", int(time.time()))
#self.set_account_attribute("last_seen", int(time.time()))
self.disconnected_since = None

def _send_status(self):
self.connection.callRemoteOnly("status", self.get_status())

def _send_account_message(self):
self.connection.callRemoteOnly("account_message", self.account_message)

def set_status(self, write, read, save):
self.can_write = write
self.can_read = read
self.can_save = save
self._send_status()

def set_account_message(self, message):
self.account_message = message
self._send_account_message()

def get_connection_status(self):
# starts as: connected=False, connected_since=None,
# last_connected_from=None, last_seen=None
Expand All @@ -251,14 +189,14 @@ def get_connection_status(self):
# after disconnect: connected=False, connected_since=None,
# last_connected_from=HOST, last_seen=STOP

last_seen = int_or_none(self.get_account_attribute("last_seen"))
last_connected_from = self.get_account_attribute("last_connected_from")
#last_seen = int_or_none(self.get_account_attribute("last_seen"))
#last_connected_from = self.get_account_attribute("last_connected_from")
created = int_or_none(self.get_account_creation_time())

return {"connected": self.connected,
"connected_since": self.connected_since,
"last_connected_from": last_connected_from,
"last_seen": last_seen,
#"last_connected_from": last_connected_from,
#"last_seen": last_seen,
"created": created,
}

Expand Down
76 changes: 7 additions & 69 deletions src/allmydata/storage/accountant.py
View file
Expand Up @@ -12,12 +12,10 @@
share a prefix with 'account.py' so my tab-autocomplete will work nicely.
"""

import simplejson, weakref
import weakref

from twisted.application import service
from foolscap.api import Referenceable

from allmydata.util import keyutil, log
from allmydata.storage.leasedb import LeaseDB, AccountingCrawler
from allmydata.storage.account import Account

Expand All @@ -29,39 +27,27 @@ def __init__(self, storage_server, dbfile, statefile):
self._leasedb = LeaseDB(dbfile)
self._active_accounts = weakref.WeakValueDictionary()
self._accountant_window = None
self._anonymous_account = Account(0, None,
self._anonymous_account = Account(LeaseDB.ANONYMOUS_ACCOUNTID, None,
self.storage_server, self._leasedb)
self._starter_account = Account(LeaseDB.STARTER_LEASE_ACCOUNTID, None,
self.storage_server, self._leasedb)

crawler = AccountingCrawler(storage_server, statefile, self._leasedb)
self._accounting_crawler = crawler
crawler.setServiceParent(self)

def get_accountant_window(self, tub):
if not self._accountant_window:
self._accountant_window = AccountantWindow(self, tub)
return self._accountant_window

def get_leasedb(self):
return self._leasedb

def set_expiration_policy(self, policy):
self._accounting_crawler.set_expiration_policy(policy)

# methods used by AccountantWindow

def get_account(self, pubkey_vs):
if pubkey_vs not in self._active_accounts:
ownernum = self._leasedb.get_or_allocate_ownernum(pubkey_vs)
a = Account(ownernum, pubkey_vs, self.storage_server, self._leasedb)
self._active_accounts[pubkey_vs] = a
# the client's RemoteReference will keep the Account alive. When
# it disconnects, that reference will lapse, and it will be
# removed from the _active_accounts WeakValueDictionary
return self._active_accounts[pubkey_vs] # note: a is still alive

def get_anonymous_account(self):
return self._anonymous_account

def get_starter_account(self):
return self._starter_account

def get_accounting_crawler(self):
return self._accounting_crawler

Expand All @@ -73,51 +59,3 @@ def get_all_accounts(self):
else:
yield Account(ownerid, pubkey_vs,
self.storage_server, self._leasedb)


class AccountantWindow(Referenceable):
def __init__(self, accountant, tub):
self.accountant = accountant
self.tub = tub

def remote_get_account(self, msg, sig, pubkey_vs):
print "GETTING ACCOUNT", msg
vk = keyutil.parse_pubkey(pubkey_vs)
vk.verify(sig, msg)
account = self.accountant.get_account(pubkey_vs)
msg_d = simplejson.loads(msg.decode("utf-8"))
rxFURL = msg_d["please-give-Account-to-rxFURL"].encode("ascii")
account.set_nickname(msg_d["nickname"])
d = self.tub.getReference(rxFURL)
def _got_rx(rx):
account.connection_from(rx)
d = rx.callRemote("account", account)
d.addCallback(lambda ign: account._send_status())
d.addCallback(lambda ign: account._send_account_message())
return d
d.addCallback(_got_rx)
d.addErrback(log.err, umid="nFYfcA")
return d


# XXX TODO new idea: move all leases into the DB. Do not store leases in
# shares at all. The crawler will exist solely to discover shares that
# have been manually added to disk (via 'scp' or some out-of-band means),
# and will add 30- or 60- day "migration leases" to them, to keep them
# alive until their original owner does a deep-add-lease and claims them
# properly. Better migration tools ('tahoe storage export'?) will create
# export files that include both the share data and the lease data, and
# then an import tool will both put the share in the right place and
# update the recipient node's lease DB.
#
# I guess the crawler will also be responsible for deleting expired
# shares, since it will be looking at both share files on disk and leases
# in the DB.
#
# So the DB needs a row per share-on-disk, and a separate table with
# leases on each bucket. When it sees a share-on-disk that isn't in the
# first table, it adds the migration-lease. When it sees a share-on-disk
# that is in the first table but has no leases in the second table (i.e.
# expired), it deletes both the share and the first-table row. When it
# sees a row in the first table but no share-on-disk (i.e. manually
# deleted share), it deletes the row (and any leases).
57 changes: 2 additions & 55 deletions src/allmydata/storage/leasedb.py
View file
Expand Up @@ -8,7 +8,7 @@
a share that has expired.
"""

import os, time, re, simplejson
import os, time, simplejson

from twisted.python.filepath import FilePath

Expand Down Expand Up @@ -136,6 +136,7 @@ def int_or_none(s):
class LeaseDB:
RETAINED_HISTORY_ENTRIES = 10

ANONYMOUS_ACCOUNTID = 0
STARTER_LEASE_ACCOUNTID = 1
STARTER_LEASE_DURATION = 2*MONTH

Expand Down Expand Up @@ -342,60 +343,6 @@ def get_history(self):
rows = self._cursor.fetchall()
return dict(rows)

# account management

def get_account_usage(self, accountid):
self._cursor.execute("SELECT SUM(`used_space`) FROM shares"
" WHERE `storage_index`, `shnum` IN"
" (SELECT DISTINCT `storage_index`, `shnum` FROM `leases`"
" WHERE `account_id`=?)",
(accountid,))
row = self._cursor.fetchone()
if not row or not row[0]: # XXX why did I need the second clause?
return 0
return row[0]

def get_account_attribute(self, accountid, name):
self._cursor.execute("SELECT `value` FROM `account_attributes`"
" WHERE `account_id`=? AND `name`=?",
(accountid, name))
row = self._cursor.fetchone()
if row:
return row[0]
return None

def set_account_attribute(self, accountid, name, value):
if self.debug: print "SET_ACCOUNT_ATTRIBUTE", accountid, name, value
self._cursor.execute("SELECT `id` FROM `account_attributes`"
" WHERE `account_id`=? AND `name`=?",
(accountid, name))
row = self._cursor.fetchone()
if row:
attrid = row[0]
self._cursor.execute("UPDATE `account_attributes`"
" SET `value`=?"
" WHERE `id`=?",
(value, attrid))
else:
self._cursor.execute("INSERT INTO `account_attributes`"
" VALUES (?,?,?,?)",
(None, accountid, name, value))
self.commit(always=True)

def get_or_allocate_ownernum(self, pubkey_vs):
if not re.search(r'^[a-zA-Z0-9+-_]+$', pubkey_vs):
raise BadAccountName("unacceptable characters in pubkey")
self._cursor.execute("SELECT `id` FROM `accounts` WHERE `pubkey_vs`=?",
(pubkey_vs,))
row = self._cursor.fetchone()
if row:
return row[0]
self._cursor.execute("INSERT INTO `accounts` VALUES (?,?,?)",
(None, pubkey_vs, int(time.time())))
accountid = self._cursor.lastrowid
self.commit(always=True)
return accountid

def get_account_creation_time(self, owner_num):
self._cursor.execute("SELECT `creation_time` from `accounts`"
" WHERE `id`=?",
Expand Down

1 comment on commit 754d326

@zooko
Copy link

@zooko zooko commented on 754d326 Oct 1, 2012

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Reviewed. +1 !! The new strategy of finishing the leasedb work, and the merge of leasedb with cloud backend, without all of this other stuff is a huge improvement. Thank you, Brian, for suggesting it, and I wish we'd thought of it before now!

Please sign in to comment.