Permalink
Browse files

Initial commit - some small parts working

  • Loading branch information...
0 parents commit e9ae4f89a44b1cac8e83ede1572f15dace692c67 @ncw committed Nov 18, 2012
Showing with 377 additions and 0 deletions.
  1. +5 −0 .gitignore
  2. +20 −0 COPYING
  3. +53 −0 README.md
  4. +32 −0 notes.txt
  5. +155 −0 swiftsync.go
  6. +112 −0 swiftsync_test.go
@@ -0,0 +1,5 @@
+*~
+*.pyc
+test-env*
+junk/
+swiftsync
@@ -0,0 +1,20 @@
+Copyright (C) 2012 by Nick Craig-Wood http://www.craig-wood.com/nick/
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
@@ -0,0 +1,53 @@
+Swiftsync
+==========
+
+Sync files and directories to and from swift
+
+FIXME
+
+
+Install
+-------
+
+Swiftsync is a Go program and comes as a single binary file.
+
+Download the relevant binary from
+
+- http://www.craig-wood.com/nick/pub/swiftsync/
+
+Or alternatively if you have Go installed use
+
+ go get github.com/ncw/swiftsync
+
+and this will build the binary in `$GOPATH/bin`. You can then modify
+the source and submit patches.
+
+Usage
+-----
+
+FIXME
+
+License
+-------
+
+This is free software under the terms of MIT the license (check the
+COPYING file included in this package).
+
+Contact and support
+-------------------
+
+The project website is at:
+
+- https://github.com/ncw/swiftsync
+
+There you can file bug reports, ask for help or contribute patches.
+
+Authors
+-------
+
+- Nick Craig-Wood <nick@craig-wood.com>
+
+Contributors
+------------
+
+- Your name goes here!
@@ -0,0 +1,32 @@
+make 100% compatible with swift.py?
+
+Make Env vars compatible with st?
+
+Get and put the metadata in the libray (x-object-meta-mtime) when getting and putting a file?
+
+st is setting this
+'x-object-meta-mtime'
+
+ getmtime(filename)
+ Return the last modification time of a file, reported by os.stat().
+
+>>> f = os.path.getmtime("z")
+1347717491.343554
+>>> print f
+1347717491.34
+>>> str(f)
+'1347717491.34'
+>>> "%d" % f
+'1347717491'
+>>>
+
+swift.py appears to be doing it wrong with str(float) which isn't a
+good way of stringifying floats...
+
+Make
+
+This also puts meta-mtime
+https://github.com/gholt/swiftly
+
+As an integer, but it does parse it as a float
+subargs.append('x-object-meta-mtime:%d' % getmtime(options.input_))
@@ -0,0 +1,155 @@
+// Sync files and directories to and from swift
+//
+// Nick Craig-Wood <nick@craig-wood.com>
+package main
+
+import (
+ //"bytes"
+ "flag"
+ "fmt"
+ //"io"
+ //"io/ioutil"
+ "log"
+ //"math/rand"
+ "os"
+ //"os/signal"
+ //"path/filepath"
+ //"regexp"
+ //"runtime"
+ "runtime/pprof"
+ "strconv"
+ "strings"
+ //"sync"
+ //"syscall"
+ //"time"
+ "github.com/ncw/swift"
+)
+
+// Globals
+var (
+ // Flags
+ //fileSize = flag.Int64("s", 1E9, "Size of the check files")
+ cpuprofile = flag.String("cpuprofile", "", "Write cpu profile to file")
+ //duration = flag.Duration("duration", time.Hour*24, "Duration to run test")
+ //statsInterval = flag.Duration("stats", time.Minute*1, "Interval to print stats")
+ //logfile = flag.String("logfile", "stressdisk.log", "File to write log to set to empty to ignore")
+
+ snet = flag.Bool("snet", false, "Use internal service network") // FIXME not implemented
+ verbose = flag.Bool("verbose", false, "Print lots more stuff")
+ quiet = flag.Bool("quiet", false, "Print as little stuff as possible")
+ // FIXME make these part of swift so we get a standard set of flags?
+ authUrl = flag.String("auth", os.Getenv("SWIFT_AUTH_USER"), "Auth URL for server. Defaults to environment var SWIFT_AUTH_USER.")
+ userName = flag.String("user", os.Getenv("ST_USER"), "User name. Defaults to environment var ST_USER.")
+ apiKey = flag.String("key", os.Getenv("ST_KEY"), "API key (password). Defaults to environment var ST_KEY.")
+)
+
+// Turns a number of ns into a floating point string in seconds
+//
+// Trims trailing zeros and guaranteed to be perfectly accurate
+func nsToFloatString(ns int64) string {
+ if ns < 0 {
+ return "-" + nsToFloatString(-ns)
+ }
+ result := fmt.Sprintf("%010d", ns)
+ split := len(result) - 9
+ result, decimals := result[:split], result[split:]
+ decimals = strings.TrimRight(decimals, "0")
+ if decimals != "" {
+ result += "."
+ result += decimals
+ }
+ return result
+}
+
+// Turns a floating point string in seconds into a ns integer
+//
+// Guaranteed to be perfectly accurate
+func floatStringToNs(s string) (ns int64, err error) {
+ if s != "" && s[0] == '-' {
+ ns, err = floatStringToNs(s[1:])
+ return -ns, err
+ }
+ point := strings.IndexRune(s, '.')
+ if point >= 0 {
+ tail := s[point+1:]
+ if len(tail) > 0 {
+ if len(tail) > 9 {
+ tail = tail[:9]
+ }
+ uns, err := strconv.ParseUint(tail, 10, 64)
+ if err != nil {
+ return 0, err
+ }
+ ns = int64(uns)
+ for i := 9 - len(tail); i > 0; i-- {
+ ns *= 10
+ }
+ }
+ s = s[:point]
+ }
+ secs, err := strconv.ParseInt(s, 10, 64)
+ if err != nil {
+ return 0, err
+ }
+ ns += int64(1000000000) * secs
+ return ns, nil
+}
+
+// syntaxError prints the syntax
+func syntaxError() {
+ fmt.Fprintf(os.Stderr, `Sync files and directores to and from swift
+
+FIXME
+
+Full options:
+`)
+ flag.PrintDefaults()
+}
+
+// Exit with the message
+func fatal(message string, args ...interface{}) {
+ syntaxError()
+ fmt.Fprintf(os.Stderr, message, args...)
+ os.Exit(1)
+}
+
+func main() {
+ flag.Usage = syntaxError
+ flag.Parse()
+ //args := flag.Args()
+ //runtime.GOMAXPROCS(3)
+
+ // Setup profiling if desired
+ if *cpuprofile != "" {
+ f, err := os.Create(*cpuprofile)
+ if err != nil {
+ log.Fatal(err)
+ }
+ pprof.StartCPUProfile(f)
+ defer pprof.StopCPUProfile()
+ }
+
+ // if len(args) < 1 {
+ // fatal("No command supplied\n")
+ // }
+
+ if *userName == "" {
+ log.Fatal("Need --user or environmental variable ST_USER")
+ }
+ if *apiKey == "" {
+ log.Fatal("Need --key or environmental variable ST_KEY")
+ }
+ if *authUrl == "" {
+ log.Fatal("Need --auth or environmental variable ST_AUTH")
+ }
+ c := swift.Connection{
+ UserName: *userName,
+ ApiKey: *apiKey,
+ AuthUrl: *authUrl,
+ }
+ err := c.Authenticate()
+ if err != nil {
+ log.Fatal("Failed to authenticate", err)
+ }
+
+}
@@ -0,0 +1,112 @@
+// Tests for swiftsync
+package main
+
+import (
+ "testing"
+)
+
+func TestNsToFloatString(t *testing.T) {
+ for _, d := range []struct {
+ ns int64
+ fs string
+ }{
+ {0, "0"},
+ {1, "0.000000001"},
+ {1000, "0.000001"},
+ {1000000, "0.001"},
+ {100000000, "0.1"},
+ {1000000000, "1"},
+ {10000000000, "10"},
+ {12345678912, "12.345678912"},
+ {12345678910, "12.34567891"},
+ {12345678900, "12.3456789"},
+ {12345678000, "12.345678"},
+ {12345670000, "12.34567"},
+ {12345600000, "12.3456"},
+ {12345000000, "12.345"},
+ {12340000000, "12.34"},
+ {12300000000, "12.3"},
+ {12000000000, "12"},
+ {10000000000, "10"},
+ {1347717491123123123, "1347717491.123123123"},
+ } {
+ if nsToFloatString(d.ns) != d.fs {
+ t.Error("Failed", d.ns, "!=", d.fs)
+ }
+ if d.ns > 0 && nsToFloatString(-d.ns) != "-"+d.fs {
+ t.Error("Failed on negative", d.ns, "!=", d.fs)
+ }
+ }
+}
+
+func TestFloatStringToNs(t *testing.T) {
+ for _, d := range []struct {
+ ns int64
+ fs string
+ }{
+ {0, "0"},
+ {0, "0."},
+ {0, "0.0"},
+ {0, "0.0000000001"},
+ {1, "0.000000001"},
+ {1000, "0.000001"},
+ {1000000, "0.001"},
+ {100000000, "0.1"},
+ {100000000, "0.10"},
+ {100000000, "0.1000000001"},
+ {1000000000, "1"},
+ {1000000000, "1."},
+ {1000000000, "1.0"},
+ {10000000000, "10"},
+ {12345678912, "12.345678912"},
+ {12345678912, "12.3456789129"},
+ {12345678912, "12.34567891299"},
+ {12345678910, "12.34567891"},
+ {12345678900, "12.3456789"},
+ {12345678000, "12.345678"},
+ {12345670000, "12.34567"},
+ {12345600000, "12.3456"},
+ {12345000000, "12.345"},
+ {12340000000, "12.34"},
+ {12300000000, "12.3"},
+ {12000000000, "12"},
+ {10000000000, "10"},
+ // This is a typical value which has more bits in than a float64
+ {1347717491123123123, "1347717491.123123123"},
+ } {
+ ns, err := floatStringToNs(d.fs)
+ if err != nil {
+ t.Error("Failed conversion", err)
+ }
+ if ns != d.ns {
+ t.Error("Failed", d.fs, "!=", d.ns, "was", ns)
+ }
+ if d.ns > 0 {
+ ns, err := floatStringToNs("-" + d.fs)
+ if err != nil {
+ t.Error("Failed conversion", err)
+ }
+ if ns != -d.ns {
+ t.Error("Failed on negative", -d.ns, "!=", "-"+d.fs)
+ }
+ }
+ }
+
+ // These are expected to produce errors
+ for _, fs := range []string{
+ "",
+ ".0",
+ " 1",
+ "- 1",
+ "- 1",
+ "1.-1",
+ "1.0.0",
+ "1x0",
+ } {
+ ns, err := floatStringToNs(fs)
+ if err == nil {
+ t.Error("Didn't produce expected error", fs, ns)
+ }
+ }
+
+}

0 comments on commit e9ae4f8

Please sign in to comment.