Permalink
Browse files

Initial commit - some small parts working

  • Loading branch information...
ncw committed Nov 18, 2012
0 parents commit e9ae4f89a44b1cac8e83ede1572f15dace692c67
Showing with 377 additions and 0 deletions.
  1. +5 −0 .gitignore
  2. +20 −0 COPYING
  3. +53 −0 README.md
  4. +32 −0 notes.txt
  5. +155 −0 swiftsync.go
  6. +112 −0 swiftsync_test.go
@@ -0,0 +1,5 @@
*~
*.pyc
test-env*
junk/
swiftsync
20 COPYING
@@ -0,0 +1,20 @@
Copyright (C) 2012 by Nick Craig-Wood http://www.craig-wood.com/nick/

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

@@ -0,0 +1,53 @@
Swiftsync
==========

Sync files and directories to and from swift

FIXME


Install
-------

Swiftsync is a Go program and comes as a single binary file.

Download the relevant binary from

- http://www.craig-wood.com/nick/pub/swiftsync/

Or alternatively if you have Go installed use

go get github.com/ncw/swiftsync

and this will build the binary in `$GOPATH/bin`. You can then modify
the source and submit patches.

Usage
-----

FIXME

License
-------

This is free software under the terms of MIT the license (check the
COPYING file included in this package).

Contact and support
-------------------

The project website is at:

- https://github.com/ncw/swiftsync

There you can file bug reports, ask for help or contribute patches.

Authors
-------

- Nick Craig-Wood <nick@craig-wood.com>

Contributors
------------

- Your name goes here!
@@ -0,0 +1,32 @@
make 100% compatible with swift.py?

Make Env vars compatible with st?

Get and put the metadata in the libray (x-object-meta-mtime) when getting and putting a file?

st is setting this
'x-object-meta-mtime'

getmtime(filename)
Return the last modification time of a file, reported by os.stat().

>>> f = os.path.getmtime("z")
1347717491.343554
>>> print f
1347717491.34
>>> str(f)
'1347717491.34'
>>> "%d" % f
'1347717491'
>>>

swift.py appears to be doing it wrong with str(float) which isn't a
good way of stringifying floats...

Make

This also puts meta-mtime
https://github.com/gholt/swiftly

As an integer, but it does parse it as a float
subargs.append('x-object-meta-mtime:%d' % getmtime(options.input_))
@@ -0,0 +1,155 @@
// Sync files and directories to and from swift
//
// Nick Craig-Wood <nick@craig-wood.com>
package main

import (
//"bytes"
"flag"
"fmt"
//"io"
//"io/ioutil"
"log"
//"math/rand"
"os"
//"os/signal"
//"path/filepath"
//"regexp"
//"runtime"
"runtime/pprof"
"strconv"
"strings"
//"sync"
//"syscall"
//"time"
"github.com/ncw/swift"
)

// Globals
var (
// Flags
//fileSize = flag.Int64("s", 1E9, "Size of the check files")
cpuprofile = flag.String("cpuprofile", "", "Write cpu profile to file")
//duration = flag.Duration("duration", time.Hour*24, "Duration to run test")
//statsInterval = flag.Duration("stats", time.Minute*1, "Interval to print stats")
//logfile = flag.String("logfile", "stressdisk.log", "File to write log to set to empty to ignore")

snet = flag.Bool("snet", false, "Use internal service network") // FIXME not implemented
verbose = flag.Bool("verbose", false, "Print lots more stuff")
quiet = flag.Bool("quiet", false, "Print as little stuff as possible")
// FIXME make these part of swift so we get a standard set of flags?
authUrl = flag.String("auth", os.Getenv("SWIFT_AUTH_USER"), "Auth URL for server. Defaults to environment var SWIFT_AUTH_USER.")
userName = flag.String("user", os.Getenv("ST_USER"), "User name. Defaults to environment var ST_USER.")
apiKey = flag.String("key", os.Getenv("ST_KEY"), "API key (password). Defaults to environment var ST_KEY.")
)

// Turns a number of ns into a floating point string in seconds
//
// Trims trailing zeros and guaranteed to be perfectly accurate
func nsToFloatString(ns int64) string {
if ns < 0 {
return "-" + nsToFloatString(-ns)
}
result := fmt.Sprintf("%010d", ns)
split := len(result) - 9
result, decimals := result[:split], result[split:]
decimals = strings.TrimRight(decimals, "0")
if decimals != "" {
result += "."
result += decimals
}
return result
}

// Turns a floating point string in seconds into a ns integer
//
// Guaranteed to be perfectly accurate
func floatStringToNs(s string) (ns int64, err error) {
if s != "" && s[0] == '-' {
ns, err = floatStringToNs(s[1:])
return -ns, err
}
point := strings.IndexRune(s, '.')
if point >= 0 {
tail := s[point+1:]
if len(tail) > 0 {
if len(tail) > 9 {
tail = tail[:9]
}
uns, err := strconv.ParseUint(tail, 10, 64)
if err != nil {
return 0, err
}
ns = int64(uns)
for i := 9 - len(tail); i > 0; i-- {
ns *= 10
}
}
s = s[:point]
}
secs, err := strconv.ParseInt(s, 10, 64)
if err != nil {
return 0, err
}
ns += int64(1000000000) * secs
return ns, nil
}

// syntaxError prints the syntax
func syntaxError() {
fmt.Fprintf(os.Stderr, `Sync files and directores to and from swift
FIXME
Full options:
`)
flag.PrintDefaults()
}

// Exit with the message
func fatal(message string, args ...interface{}) {
syntaxError()
fmt.Fprintf(os.Stderr, message, args...)
os.Exit(1)
}

func main() {
flag.Usage = syntaxError
flag.Parse()
//args := flag.Args()
//runtime.GOMAXPROCS(3)

// Setup profiling if desired
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}

// if len(args) < 1 {
// fatal("No command supplied\n")
// }

if *userName == "" {
log.Fatal("Need --user or environmental variable ST_USER")
}
if *apiKey == "" {
log.Fatal("Need --key or environmental variable ST_KEY")
}
if *authUrl == "" {
log.Fatal("Need --auth or environmental variable ST_AUTH")
}
c := swift.Connection{
UserName: *userName,
ApiKey: *apiKey,
AuthUrl: *authUrl,
}
err := c.Authenticate()
if err != nil {
log.Fatal("Failed to authenticate", err)
}

}
@@ -0,0 +1,112 @@
// Tests for swiftsync
package main

import (
"testing"
)

func TestNsToFloatString(t *testing.T) {
for _, d := range []struct {
ns int64
fs string
}{
{0, "0"},
{1, "0.000000001"},
{1000, "0.000001"},
{1000000, "0.001"},
{100000000, "0.1"},
{1000000000, "1"},
{10000000000, "10"},
{12345678912, "12.345678912"},
{12345678910, "12.34567891"},
{12345678900, "12.3456789"},
{12345678000, "12.345678"},
{12345670000, "12.34567"},
{12345600000, "12.3456"},
{12345000000, "12.345"},
{12340000000, "12.34"},
{12300000000, "12.3"},
{12000000000, "12"},
{10000000000, "10"},
{1347717491123123123, "1347717491.123123123"},
} {
if nsToFloatString(d.ns) != d.fs {
t.Error("Failed", d.ns, "!=", d.fs)
}
if d.ns > 0 && nsToFloatString(-d.ns) != "-"+d.fs {
t.Error("Failed on negative", d.ns, "!=", d.fs)
}
}
}

func TestFloatStringToNs(t *testing.T) {
for _, d := range []struct {
ns int64
fs string
}{
{0, "0"},
{0, "0."},
{0, "0.0"},
{0, "0.0000000001"},
{1, "0.000000001"},
{1000, "0.000001"},
{1000000, "0.001"},
{100000000, "0.1"},
{100000000, "0.10"},
{100000000, "0.1000000001"},
{1000000000, "1"},
{1000000000, "1."},
{1000000000, "1.0"},
{10000000000, "10"},
{12345678912, "12.345678912"},
{12345678912, "12.3456789129"},
{12345678912, "12.34567891299"},
{12345678910, "12.34567891"},
{12345678900, "12.3456789"},
{12345678000, "12.345678"},
{12345670000, "12.34567"},
{12345600000, "12.3456"},
{12345000000, "12.345"},
{12340000000, "12.34"},
{12300000000, "12.3"},
{12000000000, "12"},
{10000000000, "10"},
// This is a typical value which has more bits in than a float64
{1347717491123123123, "1347717491.123123123"},
} {
ns, err := floatStringToNs(d.fs)
if err != nil {
t.Error("Failed conversion", err)
}
if ns != d.ns {
t.Error("Failed", d.fs, "!=", d.ns, "was", ns)
}
if d.ns > 0 {
ns, err := floatStringToNs("-" + d.fs)
if err != nil {
t.Error("Failed conversion", err)
}
if ns != -d.ns {
t.Error("Failed on negative", -d.ns, "!=", "-"+d.fs)
}
}
}

// These are expected to produce errors
for _, fs := range []string{
"",
".0",
" 1",
"- 1",
"- 1",
"1.-1",
"1.0.0",
"1x0",
} {
ns, err := floatStringToNs(fs)
if err == nil {
t.Error("Didn't produce expected error", fs, ns)
}
}

}

0 comments on commit e9ae4f8

Please sign in to comment.