Skip to content

Commit

Permalink
Bug 1168890 - Implement backoff persistence and handling. r=nalexander
Browse files Browse the repository at this point in the history
  • Loading branch information
@rnewman
rnewman committed Jun 3, 2015
2 parents 1d1207d + 8fd8b01 commit 51cd63a
Show file tree
Hide file tree
Showing 9 changed files with 231 additions and 79 deletions.
36 changes: 36 additions & 0 deletions Sync/State.swift
View file
Expand Up @@ -43,7 +43,39 @@ private let PrefLastFetched = "lastFetched"
private let PrefClientName = "clientName"
private let PrefClientGUID = "clientGUID"

class PrefsBackoffStorage: BackoffStorage {
let prefs: Prefs
private let key = "backoff.server.timestamp"

init(prefs: Prefs) {
self.prefs = prefs
}

var serverBackoffUntilLocalTimestamp: Timestamp? {
get {
return self.prefs.unsignedLongForKey(self.key)
}

set(value) {
if let value = value {
self.prefs.setLong(value, forKey: self.key)
} else {
self.prefs.removeObjectForKey(self.key)
}
}
}

func clearServerBackoff() {
self.prefs.removeObjectForKey(self.key)
}

func isInBackoff(now: Timestamp) -> Timestamp? {
if let ts = self.serverBackoffUntilLocalTimestamp where now < ts {
return ts
}
return nil
}
}

/**
* The scratchpad consists of the following:
Expand Down Expand Up @@ -123,6 +155,10 @@ public class Scratchpad {
}
}

public lazy var backoffStorage: BackoffStorage = {
return PrefsBackoffStorage(prefs: self.prefs)
}()

public func evolve() -> Scratchpad.Builder {
return Scratchpad.Builder(p: self)
}
Expand Down
191 changes: 136 additions & 55 deletions Sync/StorageClient.swift
View file
Expand Up @@ -84,6 +84,28 @@ public class MalformedMetaGlobalError: ErrorType {
}
}

/**
* Raised when the storage client is refusing to make a request due to a known
* server backoff.
* If you want to bypass this, remove the backoff from the BackoffStorage that
* the storage client is using.
*/
public class ServerInBackoffError: ErrorType {
private let until: Timestamp

public var description: String {
let formatter = NSDateFormatter()
formatter.dateStyle = NSDateFormatterStyle.ShortStyle
formatter.timeStyle = NSDateFormatterStyle.MediumStyle
let s = formatter.stringFromDate(NSDate.fromTimestamp(self.until))
return "Server in backoff until \(s)."
}

public init(until: Timestamp) {
self.until = until
}
}

// Returns milliseconds. Handles decimals.
private func optionalSecondsHeader(input: AnyObject?) -> Timestamp? {
if input == nil {
Expand Down Expand Up @@ -231,62 +253,27 @@ public struct POSTResult {
public typealias Authorizer = (NSMutableURLRequest) -> NSMutableURLRequest
public typealias ResponseHandler = (NSURLRequest, NSHTTPURLResponse?, AnyObject?, NSError?) -> Void

private func errorWrap<T>(deferred: Deferred<Result<T>>, handler: ResponseHandler) -> ResponseHandler {
return { (request, response, data, error) in
log.verbose("Response is \(response).")

if let error = error {
log.error("Got error \(error). Response: \(response?.statusCode)")
deferred.fill(Result<T>(failure: RequestError(error)))
return
}

if response == nil {
// TODO: better error.
log.error("No response")
let result = Result<T>(failure: RecordParseError())
deferred.fill(result)
return
}

let response = response!

log.debug("Status code: \(response.statusCode)")
let err = StorageResponse(value: response, metadata: ResponseMetadata(response: response))

if response.statusCode >= 500 {
let result = Result<T>(failure: ServerError(err))
deferred.fill(result)
return
}

if response.statusCode == 404 {
let result = Result<T>(failure: NotFound(err))
deferred.fill(result)
return
}

if response.statusCode >= 400 {
let result = Result<T>(failure: BadRequestError(request: request, response: err))
deferred.fill(result)
return
}

handler(request, response, data, error)
}
// TODO: don't be so naïve. Use a combination of uptime and wall clock time.
public protocol BackoffStorage {
var serverBackoffUntilLocalTimestamp: Timestamp? { get set }
func clearServerBackoff()
func isInBackoff(now: Timestamp) -> Timestamp? // Returns 'until' for convenience.
}

// Don't forget to batch downloads.
public class Sync15StorageClient {
private let authorizer: Authorizer
private let serverURI: NSURL

var backoff: BackoffStorage

let workQueue: dispatch_queue_t
let resultQueue: dispatch_queue_t

public init(token: TokenServerToken, workQueue: dispatch_queue_t, resultQueue: dispatch_queue_t) {
public init(token: TokenServerToken, workQueue: dispatch_queue_t, resultQueue: dispatch_queue_t, backoff: BackoffStorage) {
self.workQueue = workQueue
self.resultQueue = resultQueue
self.backoff = backoff

// This is a potentially dangerous assumption, but failable initializers up the stack are a giant pain.
self.serverURI = NSURL(string: token.api_endpoint)!
Expand All @@ -298,11 +285,70 @@ public class Sync15StorageClient {
}
}

public init(serverURI: NSURL, authorizer: Authorizer, workQueue: dispatch_queue_t, resultQueue: dispatch_queue_t) {
public init(serverURI: NSURL, authorizer: Authorizer, workQueue: dispatch_queue_t, resultQueue: dispatch_queue_t, backoff: BackoffStorage) {
self.serverURI = serverURI
self.authorizer = authorizer
self.workQueue = workQueue
self.resultQueue = resultQueue
self.backoff = backoff
}

func updateBackoffFromResponse<T>(response: StorageResponse<T>) {
// N.B., we would not have made this request if a backoff were set, so
// we can safely avoid doing the write if there's no backoff in the
// response.
// This logic will have to change if we ever invalidate that assumption.
if let ms = response.metadata.backoffMilliseconds ?? response.metadata.retryAfterMilliseconds {
log.info("Backing off for \(ms)ms.")
self.backoff.serverBackoffUntilLocalTimestamp = ms + NSDate.now()
}
}

func errorWrap<T>(deferred: Deferred<Result<T>>, handler: ResponseHandler) -> ResponseHandler {
return { (request, response, data, error) in
log.verbose("Response is \(response).")

if let error = error {
log.error("Got error \(error). Response: \(response?.statusCode)")
deferred.fill(Result<T>(failure: RequestError(error)))
return
}

if response == nil {
// TODO: better error.
log.error("No response")
let result = Result<T>(failure: RecordParseError())
deferred.fill(result)
return
}

let response = response!

log.debug("Status code: \(response.statusCode)")
let storageResponse = StorageResponse(value: response, metadata: ResponseMetadata(response: response))

self.updateBackoffFromResponse(storageResponse)

if response.statusCode >= 500 {
let result = Result<T>(failure: ServerError(storageResponse))
deferred.fill(result)
return
}

if response.statusCode == 404 {
let result = Result<T>(failure: NotFound(storageResponse))
deferred.fill(result)
return
}

if response.statusCode >= 400 {
let result = Result<T>(failure: BadRequestError(request: request, response: storageResponse))
deferred.fill(result)
return
}

handler(request, response, data, error)
}
}

func requestGET(url: NSURL) -> Request {
Expand Down Expand Up @@ -350,10 +396,28 @@ public class Sync15StorageClient {
return self.requestWrite(url, method: Method.POST.rawValue, body: body, contentType: "application/newlines", ifUnmodifiedSince: ifUnmodifiedSince)
}

/**
* Returns true and fills the provided Deferred if our state shows that we're in backoff.
* Returns false otherwise.
*/
private func checkBackoff<T>(deferred: Deferred<Result<T>>) -> Bool {
if let until = self.backoff.isInBackoff(NSDate.now()) {
deferred.fill(Result<T>(failure: ServerInBackoffError(until: until)))
return true
}
return false
}

private func doOp<T>(op: (NSURL) -> Request, path: String, f: (JSON) -> T?) -> Deferred<Result<StorageResponse<T>>> {

let deferred = Deferred<Result<StorageResponse<T>>>(defaultQueue: self.resultQueue)

if self.checkBackoff(deferred) {
return deferred
}

let req = op(self.serverURI.URLByAppendingPathComponent(path))
let handler = errorWrap(deferred, { (_, response, data, error) in
let handler = self.errorWrap(deferred) { (_, response, data, error) in
if let json: JSON = data as? JSON {
if let v = f(json) {
let storageResponse = StorageResponse<T>(value: v, response: response!)
Expand All @@ -365,7 +429,7 @@ public class Sync15StorageClient {
}

deferred.fill(Result(failure: RecordParseError()))
})
}

req.responseParsedJSON(handler)
return deferred
Expand All @@ -380,8 +444,13 @@ public class Sync15StorageClient {
private func putResource<T>(URL: NSURL, body: JSON, ifUnmodifiedSince: Timestamp?, parser: (String) -> T?) -> Deferred<Result<StorageResponse<T>>> {

let deferred = Deferred<Result<StorageResponse<T>>>(defaultQueue: self.resultQueue)

if self.checkBackoff(deferred) {
return deferred
}

let req = self.requestPUT(URL, body: body, ifUnmodifiedSince: ifUnmodifiedSince)
let handler = errorWrap(deferred, { (_, response, data, error) in
let handler = self.errorWrap(deferred) { (_, response, data, error) in
if let data = data as? String {
if let v = parser(data) {
let storageResponse = StorageResponse<T>(value: v, response: response!)
Expand All @@ -393,7 +462,7 @@ public class Sync15StorageClient {
}

deferred.fill(Result(failure: RecordParseError()))
})
}

// Yay Swift.
let stringHandler = { (a, b, c: String?, d) in
Expand Down Expand Up @@ -467,12 +536,16 @@ public class Sync15CollectionClient<T: CleartextPayloadJSON> {
public func post(records: [Record<T>], ifUnmodifiedSince: Timestamp?) -> Deferred<Result<StorageResponse<POSTResult>>> {
let deferred = Deferred<Result<StorageResponse<POSTResult>>>(defaultQueue: client.resultQueue)

if self.client.checkBackoff(deferred) {
return deferred
}

// TODO: charset
// TODO: if any of these fail, we should do _something_. Right now we just ignore them.
let json = optFilter(records.map(self.encrypter.serializer))

let req = client.requestPOST(self.collectionURI, body: json, ifUnmodifiedSince: nil)
req.responseParsedJSON(errorWrap(deferred, { (_, response, data, error) in
req.responseParsedJSON(self.client.errorWrap(deferred) { (_, response, data, error) in
if let json: JSON = data as? JSON,
let result = POSTResult.fromJSON(json) {
let storageResponse = StorageResponse(value: result, response: response!)
Expand All @@ -482,7 +555,7 @@ public class Sync15CollectionClient<T: CleartextPayloadJSON> {
log.warning("Couldn't parse JSON response.")
}
deferred.fill(Result(failure: RecordParseError()))
}))
})

return deferred
}
Expand All @@ -499,8 +572,12 @@ public class Sync15CollectionClient<T: CleartextPayloadJSON> {
public func get(guid: String) -> Deferred<Result<StorageResponse<Record<T>>>> {
let deferred = Deferred<Result<StorageResponse<Record<T>>>>(defaultQueue: client.resultQueue)

if self.client.checkBackoff(deferred) {
return deferred
}

let req = client.requestGET(uriForRecord(guid))
req.responseParsedJSON(errorWrap(deferred, { (_, response, data, error) in
req.responseParsedJSON(self.client.errorWrap(deferred) { (_, response, data, error) in

if let json: JSON = data as? JSON {
let envelope = EnvelopeJSON(json)
Expand All @@ -515,7 +592,7 @@ public class Sync15CollectionClient<T: CleartextPayloadJSON> {
}

deferred.fill(Result(failure: RecordParseError()))
}))
})

return deferred
}
Expand All @@ -528,11 +605,15 @@ public class Sync15CollectionClient<T: CleartextPayloadJSON> {
public func getSince(since: Timestamp) -> Deferred<Result<StorageResponse<[Record<T>]>>> {
let deferred = Deferred<Result<StorageResponse<[Record<T>]>>>(defaultQueue: client.resultQueue)

if self.client.checkBackoff(deferred) {
return deferred
}

let req = client.requestGET(self.collectionURI.withQueryParams([
NSURLQueryItem(name: "full", value: "1"),
NSURLQueryItem(name: "newer", value: millisecondsToDecimalSeconds(since))]))

req.responseParsedJSON(errorWrap(deferred, { (_, response, data, error) in
req.responseParsedJSON(self.client.errorWrap(deferred) { (_, response, data, error) in

if let json: JSON = data as? JSON {
func recordify(json: JSON) -> Record<T>? {
Expand All @@ -547,7 +628,7 @@ public class Sync15CollectionClient<T: CleartextPayloadJSON> {
}

deferred.fill(Result(failure: RecordParseError()))
}))
})

return deferred
}
Expand Down

0 comments on commit 51cd63a

Please sign in to comment.