Skip to content
Browse files

Initial import

  • Loading branch information...
1 parent f6e2aee commit 4936b53cf3c563b290a9864bd7de213c745322f9 @zaitcev committed
Showing with 465 additions and 1 deletion.
  1. +1 −1 README.md
  2. +29 −0 hailcam.txt
  3. +9 −0 hailcampack.ini
  4. +231 −0 hailcampack.py
  5. +10 −0 hailcamsnap.ini
  6. +185 −0 hailcamsnap.py
View
2 README.md
@@ -1,4 +1,4 @@
hailcam
=======
-A webcam-based tool for testing object storage services such as Hail
+A webcam-based tool for testing object storage services such as Hail
View
29 hailcam.txt
@@ -0,0 +1,29 @@
+* Hailcam *
+Webcam in S3
+v.1
+
+The main element is a "stream": a (looping) collection of images and/or
+video. It usually corresponds to one webcam, but one can create a meta
+stream if desired. A stream corresponds to a key prefix in S3, so that
+all images in the stream have the same prefix. By convention hailcam
+prefix ends with a slash, although it's not mandated (SHOULD).
+
+Images and other keys are distinguished by their first letter
+after the prefix. Known types:
+
+_ - service (usually a pre-generated index for HTML browsers)
+i - image
+t - thumbnail
+p - frame (page)
+
+Clients are expected to use a prefix listing in S3 to find all keys
+in the stream. Note that it's possible to use the same prefix listing
+to find keys of a particular type.
+
+Example:
+
+${ROOTURL}hailcamtest/ <---- prefix for stream hailcamtest/
+${ROOTURL}hailcamtest/_index.html <---- service index
+${ROOTURL}hailcamtest/i19700521 <---- image (right off cam)
+${ROOTURL}hailcamtest/t19700521 <---- thumbnail for i19700521
+${ROOTURL}hailcamtest/p19700521 <---- HTML frame for i19700521
View
9 hailcampack.ini
@@ -0,0 +1,9 @@
+[pack]
+sleep = 13
+s3host = s3.zaitcev.lan
+s3user = camuser
+s3pass = campass
+s3bucket = hailcamtest
+prefix = stream0/
+maxsize = 500k
+expire = 0
View
231 hailcampack.py
@@ -0,0 +1,231 @@
+#
+# hailcampack: Part of "hailcam" that manages the stream
+#
+# Copyright (C) 2010 Red Hat, Inc.
+#
+# requires: boto, iniparse
+#
+## Parameters in hailcamsnap.ini:
+# [pack]
+# sleep = 13
+# s3host = niphredil.zaitcev.lan
+# s3user = hailcamuser
+# s3pass = hailcampass
+# s3bucket = hailcamtest
+# prefix = stream0/
+# maxsize = 12m (in 1024 units)
+# expire = 7d (in s(-econd), m(-in), h, d, w(-eek) mandatory units; 0 is never)
+
+# XXX trap the interrupt signal so it does not traceback
+
+import sys
+import os
+import time
+import rfc822
+from iniparse import ConfigParser
+from ConfigParser import NoSectionError, NoOptionError
+from boto.s3.connection import S3Connection, S3ResponseError
+# from boto.s3.bucketlistresultset import BucketListResultSet
+
+class KnownKey:
+ def __init__(self, name, size, mtime):
+ self.name = name
+ self.size = size
+ self.mtime = mtime
+
+# Attention, this sorts the keys by newest first for the expiration
+def KnownKeyCmpDate(a, b):
+ return b.mtime - a.mtime
+
+class PackScript:
+
+ def __init__(self, cfg):
+ self.cfg = cfg
+ self.known_keys = list([])
+ self.used_bytes = 0
+
+ def save_to_known(self, bucket, prefix, name):
+ if name[0] != 'i':
+ return
+
+ # The caller has a key object, so why get_key? Because the keys
+ # returned by an S3 listing may not have all attributes and meta.
+ # We fetch all the stuff by calling get_key. It's like readdir+stat.
+ # XXX Either try/except this, or take a lock against two hailcampacks.
+ key = bucket.get_key(prefix + name)
+
+ modified_tuple = rfc822.parsedate_tz(key.last_modified)
+ modified_stamp = int(rfc822.mktime_tz(modified_tuple))
+
+ # stream0/i1273784160 image/jpeg None Thu, 13 May 2010 20:56:00 +0000
+ # print key.name, key.content_type, key.filename, key.last_modified
+
+ self.known_keys.append(KnownKey(name, key.size, modified_stamp))
+
+ def expire_1(self, bucket, prefix, kkey):
+ if kkey.size + self.used_bytes > self.cfg["maxsize"]:
+ # print kkey.name, kkey.mtime, kkey.size, self.used_bytes, "Expire"
+ self.used_bytes = self.cfg["maxsize"] + 1
+ bucket.delete_key(prefix + kkey.name)
+ else:
+ # print kkey.name, kkey.mtime, kkey.size, self.used_bytes, "Keep"
+ self.used_bytes += kkey.size
+
+ # It may be ridiculously inefficient to concatenate strings so much XXX
+ def make_index(self, bucket, prefix):
+ index_name = "_index.html" # cannot start with 'i' (image)
+
+ index = ""
+ index += "<html>\r\n"
+ index += " <head>\r\n"
+ index += ' <meta http-equiv="Content-Type"' + \
+ 'content="text/html; charset=utf-8" />\r\n'
+ index += " </head>\r\n"
+
+ index += " <body>\r\n"
+
+ for kkey in self.known_keys:
+ index += ' <img src="' + kkey.name + '" /><br />\r\n'
+
+ index += " </body>\r\n"
+ index += "</html>\r\n"
+
+ mimetype = "text/html"
+ headers = { "Content-Type": mimetype }
+
+ xkey = bucket.new_key(prefix + index_name)
+ xkey.set_contents_from_string(index, headers)
+ xkey.set_acl('public-read')
+
+ def scan(self):
+ c = S3Connection(aws_access_key_id=self.cfg["s3user"],
+ aws_secret_access_key=self.cfg["s3pass"],
+ is_secure=False,
+ host=self.cfg["s3host"])
+ # socket.error: [Errno 111] Connection refused
+
+ bucketname = self.cfg["bucket"]
+ try:
+ bucket = c.get_bucket(bucketname)
+ except S3ResponseError, e:
+ # code.message is deprecated, spews a warning to stderr
+ # print >>sys.stderr, "S3ResponseError:", "code", getattr(e, 'code')
+ print >>sys.stderr, "Bucket", bucketname, "access error: %s" % e
+ bucket = None
+ # sys.exit(1)
+
+ if bucket == None:
+ try:
+ bucket = c.create_bucket(bucketname)
+ except S3CreateError, e:
+ print >>sys.stderr, "Bucket", bucketname, "create error: %s" % e
+ sys.exit(1)
+ # bucket.set_acl('public-read')
+
+ self.used_bytes = 0
+ self.known_keys = list([])
+
+ prefix = cfg["prefix"]
+ klist = bucket.list(prefix)
+ for bkey in klist:
+ self.save_to_known(bucket, prefix, bkey.name[len(prefix):])
+
+ self.known_keys.sort(KnownKeyCmpDate)
+
+ for kkey in self.known_keys:
+ self.expire_1(bucket, prefix, kkey)
+
+ self.make_index(bucket, prefix)
+
+# config()
+
+class ConfigError(Exception):
+ pass
+
+def config_size(str):
+ bytes = 0
+ num = 1
+ for c in str:
+ if num == 1:
+ if c.isdigit():
+ bytes *= 10
+ bytes += int(c)
+ else:
+ if c == 'k' or c == 'K':
+ bytes *= 1024
+ elif c == 'm' or c == 'M':
+ bytes *= 1024 * 1024
+ elif c == 'g' or c == 'G':
+ bytes *= 1024 * 1024 * 1024
+ else:
+ raise ConfigError('Invalid size ' + 'str')
+ num = 0
+ else:
+ raise ConfigError('Invalid size ' + 'str')
+ return bytes
+
+#def config_time(str):
+# return seconds
+
+# This is what cool people would do, but I don't know how to catch
+# improper syntax on this case. Se we just use ConfigParser mode.
+# from iniparse import INIConfig
+# cfgpr = INIConfig(open(cfgname))
+
+def config(cfgname, inisect):
+ cfg = { }
+ cfgpr = ConfigParser()
+ try:
+ cfgpr.read(cfgname)
+ cfg["sleep"] = cfgpr.get(inisect, "sleep")
+ cfg["s3host"] = cfgpr.get(inisect, "s3host")
+ cfg["s3user"] = cfgpr.get(inisect, "s3user")
+ cfg["s3pass"] = cfgpr.get(inisect, "s3pass")
+ cfg["bucket"] = cfgpr.get(inisect, "s3bucket")
+ cfg["prefix"] = cfgpr.get(inisect, "prefix")
+ maxsize = cfgpr.get(inisect, "maxsize")
+ #expire = cfgpr.get(inisect, "expire")
+ except NoSectionError:
+ # Unfortunately if the file does not exist, we end here.
+ raise ConfigError("Unable to open or find section " + inisect)
+ except NoOptionError, e:
+ raise ConfigError(str(e))
+
+ try:
+ cfg["sleepval"] = float(cfg["sleep"])
+ except ValueError:
+ raise ConfigError("Invalid sleep value " + cfg["sleep"])
+
+ cfg["maxsize"] = config_size(maxsize)
+ #cfg["expire"] = config_time(expire)
+
+ if maxsize == 0:
+ raise ConfigError("Zero maxsize")
+
+ if maxsize < 4096:
+ raise ConfigError("Invalid maxsize value " + cfg["maxsize"])
+
+ return cfg
+
+# main()
+
+argc = len(sys.argv)
+if argc == 1:
+ cfgname = "hailcampack.ini"
+elif argc == 2:
+ cfgname = sys.argv[1]
+else:
+ print >>sys.stderr, "Usage: hailcampack [hailcampack.ini]"
+ sys.exit(1)
+
+try:
+ cfg = config(cfgname, "pack")
+except ConfigError, e: # This is our exception. Other types traceback.
+ print >>sys.stderr, "Error in config file " + cfgname + ":", e
+ sys.exit(1)
+
+t = PackScript(cfg)
+while 1:
+ t.scan()
+ print "Sleeping %(ss)gs" % { 'ss' : cfg["sleepval"] }
+ time.sleep(cfg["sleepval"])
View
10 hailcamsnap.ini
@@ -0,0 +1,10 @@
+[snap]
+sleep = 3
+cmd = /q/zaitcev/hail/hailcam/fswebcam-tip/fswebcam -d v4l2:/dev/video0 --no-banner fs0.jpeg
+file = fs0.jpeg
+s3mode = cfk2
+s3host = kvm-rei.cloudlab.bos.bluepants.com:5000
+s3user = shared:zaitcev
+s3pass = password
+s3bucket = zaitcev-hailcam-0
+prefix = stream0/
View
185 hailcamsnap.py
@@ -0,0 +1,185 @@
+#
+# hailcamsnap: Part of "hailcam" that snaps pictures and loads into S3
+#
+# Copyright (C) 2010 Red Hat, Inc.
+#
+# requires: boto, iniparse, swiftclient
+#
+## Parameters in hailcamsnap.ini:
+# [snap]
+# sleep = 3
+# cmd = fswebcam -p BGR24 -d /dev/video0 --no-banner fs0.jpeg
+# file = fs0.jpeg (has to be a JPEG file for now; is deleted, careful)
+# s3host = niphredil.zaitcev.lan
+# s3user = hailcamsnapuser
+# s3pass = hailcamsnappass
+# s3bucket = hailcamtest
+# prefix = stream0/
+#
+# If the command is a webcam capture, this script may need to run as root.
+
+import sys
+import os
+import time
+import shlex, subprocess
+from iniparse import ConfigParser
+from ConfigParser import NoSectionError, NoOptionError
+
+from boto.s3.connection import S3Connection
+from boto.exception import S3ResponseError
+#from swiftclient import Connection, ClientException, HTTPException
+# XXX intercept exceptions
+from swiftclient import Connection
+
+class SnapScript:
+
+ def __init__(self, cfg):
+ self.cfg = cfg
+
+ def fetch(self):
+ try:
+ os.unlink(self.cfg["file"])
+ except OSError:
+ pass
+
+ cmd = self.cfg["cmd"]
+ args = shlex.split(cmd)
+ try:
+ rc = subprocess.call(args)
+ except OSError, e:
+ print >>sys.stderr, "Command", cmd, "failed:", e
+ sys.exit(1)
+
+ if rc != 0:
+ print >>sys.stderr, "Bad exit code from command", cmd, ":", rc
+ # Add a test for the file; old fswebcam often exits with zero on error
+
+ return rc
+
+ def _upload_s3(self):
+ tag = str(int(time.time()))
+
+ c = S3Connection(aws_access_key_id=self.cfg["s3user"],
+ aws_secret_access_key=self.cfg["s3pass"],
+ is_secure=False,
+ host=self.cfg["s3host"])
+ # socket.error: [Errno 111] Connection refused
+
+ # If we create a bucket here, it will be owned by the uploader, so
+ # the expiration process may fail unless it uses same ID/key. XXX
+ # Anyhow, make sure hailcampack is running somewhere, or else
+ # the below with fail with a missing bucket.
+ bucketname = self.cfg["bucket"]
+ try:
+ bucket = c.get_bucket(bucketname)
+ except S3ResponseError, e:
+ # code.message is deprecated, spews a warning to stderr
+ # print >>sys.stderr, "S3ResponseError:", "code", getattr(e, 'code')
+ print >>sys.stderr, "Bucket", bucketname, "access error: %s" % e
+ sys.exit(1)
+
+ key = bucket.new_key()
+
+ key.name = self.cfg["prefix"] + '/' + "i" + tag
+ mimetype = "image/jpeg"
+ headers = { "Content-Type": mimetype }
+ # key.set_contents_from_filename(self.cfg["file"])
+ fp = open(self.cfg["file"], 'rb')
+ key.set_contents_from_file(fp, headers)
+ fp.close()
+ key.set_acl('public-read')
+
+ # and now we just return and c is garbage-collected.
+
+ def _upload_swift(self):
+ tag = str(int(time.time()))
+
+ (tenant, user) = self.cfg["s3user"].split(':')
+ auth_url = "http://" + self.cfg["s3host"] + "/v2.0/"
+
+ conn = Connection(auth_url, user, self.cfg["s3pass"],
+ snet=False, tenant_name=tenant, auth_version="2")
+
+ # Not calling conn.put_container() for the same reason of permissions.
+
+ fp = open(self.cfg["file"], 'rb')
+ conn.put_object(self.cfg["bucket"], 'i'+tag, fp)
+ fp.close()
+
+ def upload(self):
+ if self.cfg["s3mode"] == 'cfk2':
+ # Swift with Keystone authentication
+ self._upload_swift()
+ else:
+ # Amazon S3
+ self._upload_s3()
+
+# config()
+
+class ConfigError(Exception):
+ pass
+
+# This is what cool people would do, but I don't know how to catch
+# improper syntax on this case. Se we just use ConfigParser mode.
+# from iniparse import INIConfig
+# cfgpr = INIConfig(open(cfgname))
+
+def config(cfgname, inisect):
+ cfg = { }
+ cfgpr = ConfigParser()
+ try:
+ cfgpr.read(cfgname)
+ cfg["sleep"] = cfgpr.get(inisect, "sleep")
+ cfg["cmd"] = cfgpr.get(inisect, "cmd")
+ cfg["file"] = cfgpr.get(inisect, "file")
+ cfg["s3mode"] = cfgpr.get(inisect, "s3mode")
+ cfg["s3host"] = cfgpr.get(inisect, "s3host")
+ cfg["s3user"] = cfgpr.get(inisect, "s3user")
+ cfg["s3pass"] = cfgpr.get(inisect, "s3pass")
+ cfg["bucket"] = cfgpr.get(inisect, "s3bucket")
+ cfg["prefix"] = cfgpr.get(inisect, "prefix")
+ except NoSectionError:
+ # Unfortunately if the file does not exist, we end here.
+ raise ConfigError("Unable to open or find section " + inisect)
+ except NoOptionError, e:
+ raise ConfigError(str(e))
+
+ try:
+ cfg["sleepval"] = float(cfg["sleep"])
+ except ValueError:
+ raise ConfigError("Invalid sleep value " + cfg["sleep"])
+
+ if cfg["s3mode"] == 'cfk2' and len(cfg["s3user"].split(':')) == 1:
+ raise ConfigError("Must have a ':' in user " + cfg["s3user"])
+
+ return cfg
+
+# main()
+# def main(args):
+
+argc = len(sys.argv)
+if argc == 1:
+ cfgname = "hailcamsnap.ini"
+elif argc == 2:
+ cfgname = sys.argv[1]
+else:
+ print >>sys.stderr, "Usage: hailcamsnap [hailcamsnap.ini]"
+ sys.exit(1)
+
+try:
+ cfg = config(cfgname, "snap")
+except ConfigError, e: # This is our exception. Other types traceback.
+ print >>sys.stderr, "Error in config file " + cfgname + ":", e
+ sys.exit(1)
+
+t = SnapScript(cfg)
+while 1:
+ rc = t.fetch()
+ if rc == 0:
+ t.upload()
+ print "Sleeping %(ss)gs" % { 'ss' : cfg["sleepval"] }
+ time.sleep(cfg["sleepval"])
+
+## http://utcc.utoronto.ca/~cks/space/blog/python/ImportableMain
+#if __name__ == "__main__":
+# main(sys.argv[1:])

0 comments on commit 4936b53

Please sign in to comment.
Something went wrong with that request. Please try again.