From e0c1cc34aa2efbff88073e3fa075d1474710e752 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Thu, 2 Dec 2010 21:26:25 -0600 Subject: [PATCH] Undo the undo commit 39b3554156ee8c368bdc0db5fe98939815c94c67 {sigh} Apparently I need to read graphs a bit more closely. Or tell the difference between left & right. Or merge the proper branch X into the proper branch Y before pushing to the outside world. Or something. Translation: restore Jon's bctt script and Ryan's packaging changes. --- Makefile | 40 ++- package/Makefile | 38 +++ package/deb/Makefile | 28 ++ package/deb/bitcask.debhelper.log | 6 + package/deb/bitcask.substvars | 1 + package/deb/changelog | 5 + package/deb/compat | 1 + package/deb/control | 13 + package/deb/copyright | 32 ++ package/deb/dirs | 1 + package/deb/files | 1 + package/deb/postrm | 42 +++ package/deb/rules | 48 +++ package/rpm/Makefile | 31 ++ package/rpm/SPECS/bitcask.spec | 55 ++++ package/solaris/Makefile | 49 ++++ package/solaris/copyright | 178 +++++++++++ package/solaris/depend | 5 + package/solaris/pkginfo.tmpl | 13 + test/bctt | 472 ++++++++++++++++++++++++++++++ 20 files changed, 1050 insertions(+), 9 deletions(-) create mode 100644 package/Makefile create mode 100644 package/deb/Makefile create mode 100644 package/deb/bitcask.debhelper.log create mode 100644 package/deb/bitcask.substvars create mode 100644 package/deb/changelog create mode 100644 package/deb/compat create mode 100644 package/deb/control create mode 100644 package/deb/copyright create mode 100644 package/deb/dirs create mode 100644 package/deb/files create mode 100644 package/deb/postrm create mode 100644 package/deb/rules create mode 100644 package/rpm/Makefile create mode 100644 package/rpm/SPECS/bitcask.spec create mode 100644 package/solaris/Makefile create mode 100644 package/solaris/copyright create mode 100644 package/solaris/depend create mode 100644 package/solaris/pkginfo.tmpl create mode 100755 test/bctt diff --git a/Makefile b/Makefile index 367a7206..a5d1c1a8 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,9 @@ -.PHONY: rel deps +REPO ?= bitcask +BITCASK_TAG = $(shell git describe --tags) +REVISION ?= $(shell echo $(BITCASK_TAG) | sed -e 's/^$(REPO)-//') +PKG_VERSION ?= $(shell echo $(REVISION) | tr - .) -BITCASK_TAG = $(shell hg identify -t) +.PHONY: rel deps package pkgclean all: deps compile @@ -15,18 +18,37 @@ clean: # Release tarball creation # Generates a tarball that includes all the deps sources so no checkouts are necessary - +archivegit = git archive --format=tar --prefix=$(1)/ HEAD | (cd $(2) && tar xf -) +archivehg = hg archive $(2)/$(1) +archive = if [ -d ".git" ]; then \ + $(call archivegit,$(1),$(2)); \ + else \ + $(call archivehg,$(1),$(2)); \ + fi + +buildtar = mkdir distdir && \ + git clone . distdir/$(REPO)-clone && \ + cd distdir/$(REPO)-clone && \ + git checkout $(BITCASK_TAG) && \ + $(call archive,$(BITCASK_TAG),..) && \ + mkdir ../$(BITCASK_TAG)/deps && \ + make deps; \ + for dep in deps/*; do cd $${dep} && $(call archive,$${dep},../../../$(BITCASK_TAG)); cd ../..; done + distdir: - $(if $(findstring tip,$(BITCASK_TAG)),$(error "You can't generate a release tarball from tip")) - mkdir distdir - hg clone -u $(BITCASK_TAG) . distdir/bitcask-clone - cd distdir/bitcask-clone; \ - hg archive ../$(BITCASK_TAG) + $(if $(BITCASK_TAG), $(call buildtar), $(error "You can't generate a release tarball from a non-tagged revision. Run 'git checkout ', then 'make dist'")) dist $(BITCASK_TAG).tar.gz: distdir cd distdir; \ tar czf ../$(BITCASK_TAG).tar.gz $(BITCASK_TAG) -distclean: +ballclean: rm -rf $(BITCASK_TAG).tar.gz distdir +package: dist + $(MAKE) -C package package + +pkgclean: + $(MAKE) -C package pkgclean + +export BITCASK_TAG PKG_VERSION REPO REVISION diff --git a/package/Makefile b/package/Makefile new file mode 100644 index 00000000..b39a75a8 --- /dev/null +++ b/package/Makefile @@ -0,0 +1,38 @@ +OS = $(shell uname -s) +KERNEL = $(shell uname -r) +ifeq ($(OS),Linux) +PKGER = $(shell cat /etc/redhat-release 2> /dev/null) +ifeq ($(PKGER),) +PKGER = debuild +PKGERDIR = deb +else +PKGER = rpmbuild +PKGERDIR = rpm +endif +endif +ifeq ($(OS),SunOS) +PKGER = make +PKGERDIR = solaris +DISTRO = $(shell awk '{ if (NR==1) print $$1; };' /etc/release) +endif + +APP = $(shell echo "$(REPO)" | sed -e 's/_/-/g') +# Assumes CURDIR is $(REPO)/package/ +RIAK_PATH ?= .. +RELEASE ?= + +$(APP)-$(REVISION).tar.gz: ../$(BITCASK_TAG).tar.gz + ln -s $< $@ + +pkgclean: $(PKGERDIR)/pkgclean + rm -rf $(APP)-$(REVISION).tar.gz working rpmbuild debuild packages + +pkgcheck: + $(if $(BITCASK_TAG),,$(error "You can't generate a release tarball from a non-tagged revision. Run 'git checkout ', then 'make dist'")) + $(if $(RELEASE),,$(error "You must provide a package release number via RELEASE= on the command line")) + @echo "Packaging \"$(BITCASK_TAG)\"" + +# The heavy lifting is done by the individual packager Makefiles +package: pkgcheck build + +include $(PKGERDIR)/Makefile diff --git a/package/deb/Makefile b/package/deb/Makefile new file mode 100644 index 00000000..823a9a86 --- /dev/null +++ b/package/deb/Makefile @@ -0,0 +1,28 @@ +BUILDPATH = debuild/$(APP)-$(REVISION) + +build: $(BUILDPATH)/debian \ + debuild/$(APP)_$(REVISION).orig.tar.gz + export DEBFULLNAME="Basho Buildbot Packager"; \ + export DEBEMAIL="support@basho.com"; \ + dch --noquery -c $(BUILDPATH)/debian/changelog \ + -b -v "$(REVISION)-$(RELEASE)" "pants on head" + cd $(BUILDPATH) && debuild --no-lintian \ + -e REVISION=$(REVISION) \ + -e RELEASE=$(RELEASE) \ + -uc -us + mkdir -p packages + mv debuild/$(APP)_$(REVISION)-$(RELEASE)_*.deb packages + +$(BUILDPATH): $(APP)-$(REVISION).tar.gz + mkdir -p debuild + tar xz -C debuild -f $^ + +$(BUILDPATH)/debian: $(BUILDPATH) + cp -a $(PKGERDIR) $@ + rm -rf $@/.hg $@/Makefile $@/.*.swp + +debuild/$(APP)_$(REVISION).orig.tar.gz: $(APP)-$(REVISION).tar.gz + cp $^ $@ + +$(PKGERDIR)/pkgclean: + @echo diff --git a/package/deb/bitcask.debhelper.log b/package/deb/bitcask.debhelper.log new file mode 100644 index 00000000..a9f4eea9 --- /dev/null +++ b/package/deb/bitcask.debhelper.log @@ -0,0 +1,6 @@ +dh_installdirs +dh_strip +dh_compress +dh_installdeb +dh_gencontrol +dh_builddeb diff --git a/package/deb/bitcask.substvars b/package/deb/bitcask.substvars new file mode 100644 index 00000000..abd3ebeb --- /dev/null +++ b/package/deb/bitcask.substvars @@ -0,0 +1 @@ +misc:Depends= diff --git a/package/deb/changelog b/package/deb/changelog new file mode 100644 index 00000000..14cf84e9 --- /dev/null +++ b/package/deb/changelog @@ -0,0 +1,5 @@ +bitcask (8) unstable; urgency=low + + * Initial release of bitcask key/value store + + -- Ryan Tilder Wed, 26 May 2010 16:20:40 UTC diff --git a/package/deb/compat b/package/deb/compat new file mode 100644 index 00000000..7f8f011e --- /dev/null +++ b/package/deb/compat @@ -0,0 +1 @@ +7 diff --git a/package/deb/control b/package/deb/control new file mode 100644 index 00000000..e0166db2 --- /dev/null +++ b/package/deb/control @@ -0,0 +1,13 @@ +Source: bitcask +Section: net +Priority: extra +Maintainer: Basho Support +Build-Depends: debhelper (>= 7) +Standards-Version: 3.8.3 +Homepage: http://basho.com/ + +Package: bitcask +Architecture: any +Depends: adduser, logrotate, ${shlibs:Depends}, ${misc:Depends} +Description: Because you need another a key/value storage engine + Because you need another a key/value storage engine diff --git a/package/deb/copyright b/package/deb/copyright new file mode 100644 index 00000000..c1431e5a --- /dev/null +++ b/package/deb/copyright @@ -0,0 +1,32 @@ +This package was debianized by Basho Support on +Wed, 26 May 2010 16:20:40 UTC + +It was downloaded from http://downloads.basho.com/bitcask/bitcask-0.1/ + +Upstream Author(s): + + +Copyright: + 2007-2010 Basho Technologies + +License: + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +The Debian packaging is: + + 2007-2010 Basho Technologies + +and is licensed under the Apache License, Version 2.0 + +# Please also look if there are files or directories which have a +# different copyright/license attached and list them here. diff --git a/package/deb/dirs b/package/deb/dirs new file mode 100644 index 00000000..5128301a --- /dev/null +++ b/package/deb/dirs @@ -0,0 +1 @@ +usr/lib/riak/lib/bitcask-0.1 diff --git a/package/deb/files b/package/deb/files new file mode 100644 index 00000000..24f98fa0 --- /dev/null +++ b/package/deb/files @@ -0,0 +1 @@ +bitcask_0.1_i386.deb net extra diff --git a/package/deb/postrm b/package/deb/postrm new file mode 100644 index 00000000..957bf212 --- /dev/null +++ b/package/deb/postrm @@ -0,0 +1,42 @@ +#!/bin/sh +# postrm script for riak +# +# see: dh_installdeb(1) + + +# summary of how this script can be called: +# * `remove' +# * `purge' +# * `upgrade' +# * `failed-upgrade' +# * `abort-install' +# * `abort-install' +# * `abort-upgrade' +# * `disappear' +# +# for details, see http://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + purge) + if [ -d /var/lib/riak/lib/bitcask-0.1 ]; then + rm -r /var/lib/riak/lib/bitcask-0.1 + fi + ;; + + remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) + ;; + + *) + echo "postrm called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/package/deb/rules b/package/deb/rules new file mode 100644 index 00000000..adea56af --- /dev/null +++ b/package/deb/rules @@ -0,0 +1,48 @@ +#!/usr/bin/make -f +# -*- makefile -*- +# Sample debian/rules that uses debhelper. +# This file was originally written by Joey Hess and Craig Small. +# As a special exception, when this file is copied by dh-make into a +# dh-make output file, you may use that output file without restriction. +# This special exception was added by Craig Small in version 0.37 of dh-make. + +# Uncomment this to turn on verbose mode. +export DH_VERBOSE=1 + +package=bitcask + +CFLAGS= +LDFLAGS= + + +build: + ERL_FLAGS="-smp enable" make + touch build + +clean: + dh_clean + rm -f build + + # Add here commands to clean up after the build process. + make clean + +install: build + dh_testdir + dh_testroot + dh_installdirs + cp -R src debian/$(package)/usr/lib/riak/lib/$(package)-$(REVISION) + cp -R ebin debian/$(package)/usr/lib/riak/lib/$(package)-$(REVISION) + cp -R priv debian/$(package)/usr/lib/riak/lib/$(package)-$(REVISION) + +binary-indep: install +# We have nothing to do by default. + +# Build architecture-dependent files here. +binary-arch: install + dh_strip -a + dh_compress -a + dh_installdeb + dh_gencontrol + dh_builddeb + +binary: binary-indep binary-arch diff --git a/package/rpm/Makefile b/package/rpm/Makefile new file mode 100644 index 00000000..eb6171e5 --- /dev/null +++ b/package/rpm/Makefile @@ -0,0 +1,31 @@ + +build: $(PKGERDIR)/SOURCES/$(APP)-$(REVISION).tar.gz rpmbuild + @echo "BITCASK_TAG = $(BITCASK_TAG)" + @echo "REVISION = $(REVISION)" + @echo "RELEASE = $(RELEASE)" + rpmbuild --define '_topdir $(CURDIR)/rpmbuild' \ + --define '_sourcedir $(CURDIR)/$(PKGERDIR)/SOURCES' \ + --define '_specdir $(CURDIR)/$(PKGERDIR)/SPECS' \ + --define '_rpmdir $(CURDIR)/packages' \ + --define '_srcrpmdir $(CURDIR)/packages' \ + --define "_revision $(REVISION)" \ + --define "_version $(PKG_VERSION)" \ + --define "_release $(RELEASE)" \ + -ba $(PKGERDIR)/SPECS/$(APP).spec + mv packages/*/$(APP)-$(PKG_VERSION)-$(RELEASE)*.rpm packages + rm -rf packages/i?86 packages/x86_64 + +rpmbuild: + @mkdir -p rpmbuild/BUILD + @mkdir -p packages + +# In case it doesn't exist because there aren't any patches to apply +$(PKGERDIR)/SOURCES: + @mkdir -m 0755 -p $(PKGERDIR)/SOURCES + +$(PKGERDIR)/SOURCES/$(APP)-$(REVISION).tar.gz: $(APP)-$(REVISION).tar.gz \ + $(PKGERDIR)/SOURCES + cp $(APP)-$(REVISION).tar.gz $(PKGERDIR)/SOURCES + +$(PKGERDIR)/pkgclean: + @echo diff --git a/package/rpm/SPECS/bitcask.spec b/package/rpm/SPECS/bitcask.spec new file mode 100644 index 00000000..fa3197a2 --- /dev/null +++ b/package/rpm/SPECS/bitcask.spec @@ -0,0 +1,55 @@ +# BuildArch should be determined automatically. Use of setarch on x86-64 +# platform will allow one to build ix86 only +# +# _revision, _release, and _version should be defined on the rpmbuild command +# line like so: +# +# --define "_version 0.9.1.19.abcdef" --define "_release 7" \ +# --define "_revision 0.9.1-19-abcdef" + + +Name: bitcask +Version: %{_version} +Release: %{_release}%{?dist} +License: GPLv2 +Group: Development/Libraries +Source: http://downloads.basho.com/%{name}/%{name}-%{_revision}/%{name}-%{_revision}.tar.gz +URL: http://basho.com/ +Vendor: Basho Technologies +Packager: Basho Support +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root +Summary: Because you need another a key/value storage engine + +%description +Because you need another a key/value storage engine + +%define __prelink_undo_cmd /bin/cat prelink library + +%prep +%setup -n %{name}-%{_revision} + +%build +mkdir %{name} +ERL_FLAGS="-smp enable" make + +%install +mkdir -p %{buildroot}%{_libdir}%{name} + +#Copy all necessary lib files etc. +cp -r $RPM_BUILD_DIR/%{name}-%{_revision}/ebin %{buildroot}%{_libdir}%{name} +cp -r $RPM_BUILD_DIR/%{name}-%{_revision}/priv %{buildroot}%{_libdir}%{name} +# I don't see how the source is useful at the moment +#cp -r $RPM_BUILD_DIR/%{name}-%{_revision}/src %{buildroot}%{_libdir}%{name} + +%files +%defattr(-,root,root) +%dir %{_libdir}%{name} +%{_libdir}%{name}/* + +%clean +rm -rf %{buildroot} + +%changelog +* Wed May 26 2010 Ryan Tilder 0.1-1 +- Initial packaging + diff --git a/package/solaris/Makefile b/package/solaris/Makefile new file mode 100644 index 00000000..f4f351c9 --- /dev/null +++ b/package/solaris/Makefile @@ -0,0 +1,49 @@ +# requires GNU make +PKG = BASHO$(APP) +# possible ARCH values are i386, sparc, all +ARCH = $(shell uname -p) +SOLARIS_VER ?= $(shell echo "$(KERNEL)" | sed -e 's/^5\.//') +PKGFILE = $(PKG)-$(REVISION)-$(RELEASE)-$(DISTRO)$(SOLARIS_VER)-$(ARCH).pkg +BITCASK_LIB ?= lib/$(APP)-$(REVISION) + +build: buildrel pkginfo prototype + cp $(PKGERDIR)/copyright $(PKGERDIR)/depend . + mkdir -p pkgbuild packages + pkgmk -o -d pkgbuild -a $(ARCH) + touch packages/$(PKGFILE) + pkgtrans -s pkgbuild packages/$(PKGFILE) $(PKG) + rm -r pkgbuild/$(PKG) + gzip packages/$(PKGFILE) + @echo + @echo Wrote: $(CURDIR)/packages/$(PKGFILE).gz + @echo + +# Build the release we need to package +buildrel: + @# Ye Olde Bourne Shell on Solaris means we have to do it old school + PATH=/opt/erlang/R13B04-$(DISTRO)$(SOLARIS_VER):$${PATH}; \ + export PATH; \ + echo "Using `which erl` to build"; \ + $(MAKE) -C $(BITCASK_PATH) + +pkginfo: + sed -e 's/@@VERSION@@/$(REVISION)-$(RELEASE)/g' \ + -e 's/@@PKG@@/$(PKG)/g' \ + -e 's/@@PKGNAME@@/$(APP)/g' \ + < $(PKGERDIR)/pkginfo.tmpl > pkginfo + +# NOTE! The instances of riak below shouldn't change +prototype: + echo "i pkginfo" > prototype + echo "i copyright" >> prototype + echo "i depend" >> prototype + echo '' >> prototype + echo "d none lib/$(APP)-$(REVISION) 0755 riak riak" >> prototype + pkgproto $(BITCASK_PATH)/ebin=$(BITCASK_LIB)/ebin >> prototype + pkgproto $(BITCASK_PATH)/priv=$(BITCASK_LIB)/priv >> prototype + sed -i -e 's/basho other/riak riak/' \ + -e 's/buildbot other/riak riak/' prototype + +$(PKGERDIR)/pkgclean: + rm -rf copyright depend pkgbuild pkginfo prototype + diff --git a/package/solaris/copyright b/package/solaris/copyright new file mode 100644 index 00000000..e454a525 --- /dev/null +++ b/package/solaris/copyright @@ -0,0 +1,178 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + diff --git a/package/solaris/depend b/package/solaris/depend new file mode 100644 index 00000000..6d100950 --- /dev/null +++ b/package/solaris/depend @@ -0,0 +1,5 @@ +# Same dependencies as Erlang +P SUNWlibmsr Math & Microtasking Libraries (Root) +P SUNWlibms Math & Microtasking Libraries (Usr) +P SUNWopensslr OpenSSL (Root) +P SUNWopenssl-libraries OpenSSL Libraries (Usr) diff --git a/package/solaris/pkginfo.tmpl b/package/solaris/pkginfo.tmpl new file mode 100644 index 00000000..979ac7f1 --- /dev/null +++ b/package/solaris/pkginfo.tmpl @@ -0,0 +1,13 @@ +# Set this for the default basedir for relocatable packages +BASEDIR=/opt/riak +CLASSES=none +TZ=EST +PATH=/sbin:/usr/sbin:/usr/bin:/usr/sadm/install/bin +PKG=@@PKG@@ +NAME=@@PKGNAME@@ +VERSION=@@VERSION@@ +CATEGORY=application +DESC=Because you need another a key/value storage engine +VENDOR=Basho Technologies +EMAIL=riak@basho.com +PKGSAV=/var/sadm/pkg/@@PKG@@/save diff --git a/test/bctt b/test/bctt new file mode 100755 index 00000000..16bc16f4 --- /dev/null +++ b/test/bctt @@ -0,0 +1,472 @@ +#! /usr/bin/env escript +%% -*- erlang -*- +%% +%% Bitcask Torture Tests +%% +%% Pre-populates a bitcask file with a known range of keys, the value for +%% each key stores the key itself and a sequence number for the write +%% (initially 1). +%% +%% The test itself consists of a writer continually rewriting all keys with +%% the next sequence number while other processes read, fold over and merge +%% the cask. +%% + +-module(bctt). +-compile([export_all]). + +-record(state, {seq = 0, + reader_reps=0, + folder_reps=0, + merger_reps=0, + duration, + status_freq = 10000, + cask = "bctt.bc", + num_keys=16384, + writer_pid, + restart_writer=false, + writers=1, + readers=0, + folders=0, + mergers=0, + max_file_size=10*1024*1024}). + +%% Function to run from inside beam +test() -> + test([]). + +test(Opts) -> + {ok, State} = process_args(Opts, #state{}), + print_params(State), + spawn(fun() -> do_test(State) end). + +%% Escript version +main() -> + main([]). + +main(Args) -> + try + case process_args([parse_arg(Arg) || Arg <- Args], #state{}) of + {ok, State} -> + print_params(State), + ensure_bitcask(), + ensure_deps(), + do_test(State); + syntax -> + syntax(), + halt(0) + end + catch + _:Why -> + io:format("Failed: ~p\n", [Why]), + halt(1) + end. + +syntax() -> + io:format("bctt [duration=Msecs]\n" + " [cask=CaskFile]\n" + " [num_keys=NumKeys]\n" + " [readers=NumReaders]\n" + " [folders=NumFolders]\n" + " [mergers=NumMergers]\n" + " [max_file_size=Bytes] # set 0 for defaults\n" + " [restart_writer=true|false]\n"). + + +do_test(State0) -> + erlang:process_flag(trap_exit, true), + os:cmd("rm -rf " ++ State0#state.cask), + + %% Put a base of keys - each value has {Key, Seq}, starting from 1. + io:format("\nInitial populate.\n"), + State1 = start_writer(State0), + kick_writer(State1), + wait_for_writer(State1), + + %% Start continually rewriting the keys and optionally reading, + %% folding and merging + State = State1#state{seq = 1}, + kick_writer(State), + start_readers(State, State#state.readers), + start_folders(State, State#state.folders), + start_mergers(State, State#state.mergers), + schedule_status(State), + case State#state.duration of + undefined -> + self() ! stop; + Duration -> + timer:send_after(Duration, self(), stop) + end, + io:format("Starting test.\n"), + EndState = restart_procs(State), + stop_writer(EndState), + wait_for_procs(EndState). + +restart_procs(State) -> + receive + stop -> + io:format("Test ending...\n"), + State; + status -> + io:format("Writer seq: ~p Readers=~p Folders=~p Merges=~p\n", + [State#state.seq, State#state.reader_reps, + State#state.folder_reps, State#state.merger_reps]), + schedule_status(State), + restart_procs(State); + {write_done, WriteSeq} -> + true = (State#state.seq+1 =:= WriteSeq), + NewState = State#state{seq = WriteSeq}, + case State#state.restart_writer of + true -> + stop_writer(State); + false -> + kick_writer(NewState) + end, + restart_procs(NewState); + write_exit -> + State1 = start_writer(State), + kick_writer(State1), + restart_procs(State1); + merge_done -> + start_mergers(State, 1), + restart_procs(State#state{merger_reps = State#state.merger_reps+1}); + {read_done, _ReadSeq} -> + start_readers(State, 1), + restart_procs(State#state{reader_reps = State#state.reader_reps+1}); + {fold_done, _FoldSeq} -> + start_folders(State, 1), + restart_procs(State#state{folder_reps = State#state.folder_reps+1}); + {'EXIT', _From, normal} -> + restart_procs(State); + {'EXIT', From, Reason} -> + io:format("Test process ~p died\n~p\n", [From, Reason]), + State; + Other -> + io:format("Restart procs got unexpected message\n~p\n", [Other]), + State + end. + +%% Wait for the initial writer to complete - the os:cmd call +%% can generate an EXIT message +wait_for_writer(State) -> + WriterPid = State#state.writer_pid, + receive + {'EXIT', WriterPid, Why} -> + erlang:error({initial_write_failed, Why}); + {'EXIT', _Pid, _Why} -> + wait_for_writer(State); + {write_done, 1} -> + ok + end. + +wait_for_procs(#state{writers = 0, readers = 0, folders = 0, mergers = 0}) -> + io:format("Test complete\n"); +wait_for_procs(State) -> + receive + status -> + wait_for_procs(State); + {read_done, _ReadSeq} -> + wait_for_procs(State#state{readers = State#state.readers - 1}); + {fold_done, _FoldSeq} -> + wait_for_procs(State#state{folders = State#state.folders - 1}); + merge_done -> + wait_for_procs(State#state{mergers = State#state.mergers - 1}); + {write_done, _WriteSeq} -> + wait_for_procs(State); + write_exit -> + wait_for_procs(State#state{writers = State#state.writers - 1}); + {'EXIT', _From, normal} -> + wait_for_procs(State); + {'EXIT', From, Reason} -> + io:format("Test process ~p died\n~p\n", [From, Reason]); + Other -> + io:format("Wait for procs got unexpected message\n~p\n", [Other]) + end. + +schedule_status(State) -> + case State#state.status_freq of + undefined -> + ok; + StatusFreq -> + timer:send_after(StatusFreq, self(), status) + end. + +start_writer(State) -> + Caller = self(), + Pid = spawn_link(fun() -> + Opts = writer_opts(State), + Ref = bitcask:open(State#state.cask, Opts), + write_proc(Ref, Caller) + end), +%%X io:format("Started writer pid ~p\n", [Pid]), + State#state{writer_pid = Pid}. + +writer_opts(State) -> + lists:flatten( + [read_write], + case State#state.max_file_size of + Size when is_integer(Size), Size > 0-> + [{max_file_size, Size}]; + _ -> + [] + end). + +start_readers(_State, 0) -> + ok; +start_readers(State, NumReaders) -> + Caller = self(), + spawn_link(fun() -> + read_proc(State#state.cask, State#state.num_keys, + State#state.seq, Caller) + end), + start_readers(State, NumReaders - 1). + +start_folders(_State, 0) -> + ok; +start_folders(State, NumFolders) -> + Caller = self(), + spawn_link(fun() -> + fold_proc(State#state.cask, State#state.num_keys, + State#state.seq, Caller) + end), + start_folders(State, NumFolders - 1). + +start_mergers(_State, 0) -> + ok; +start_mergers(State, NumMergers) -> + Caller = self(), + spawn_link(fun() -> merge_proc(State#state.cask, Caller) end), + start_mergers(State, NumMergers - 1). + +kick_writer(State) -> + State#state.writer_pid ! {start, State#state.seq + 1, State#state.num_keys}. + +stop_writer(State) -> +%%X io:format("Stopping writer ~p\n", [State#state.writer_pid]), + MRef = erlang:monitor(process, State#state.writer_pid), + State#state.writer_pid ! stop, + receive + {'DOWN', MRef, _, _, _} -> +%%X io:format("Stopped writer ~p\n", [State#state.writer_pid]), + ok + after + 10000 -> + erlang:error({writer_pid_timeout, State#state.writer_pid}) + end. + +write_proc(Ref, Caller) -> + receive + stop -> +%%X io:format("Writer ~p received stop request\n", [self()]), + Caller ! write_exit; + {start, Seq, NumKeys} -> + write(Ref, NumKeys, Seq), + Caller ! {write_done, Seq}, + write_proc(Ref, Caller) + end. + +write(_Ref, 0, _Seq) -> + ok; +write(Ref, Key, Seq) -> + bitcask:put(Ref, <>, term_to_binary({Key, Seq})), + write(Ref, Key - 1, Seq). + +read_proc(Cask, NumKeys, Seq, Caller) -> + Ref = bitcask:open(Cask), + read(Ref, NumKeys, Seq), + bitcask:close(Ref), + Caller ! {read_done, Seq}. + +read(_Ref, 0, _Iter) -> + ok; +read(Ref, Key, MinSeq) -> + {ok, Bin} = bitcask:get(Ref, <>), + {Key, Seq} = binary_to_term(Bin), + true = (Seq >= MinSeq), + read(Ref, Key - 1, MinSeq). + +fold_proc(Cask, NumKeys, Seq, Caller) -> + Ref = bitcask:open(Cask), + fold(Ref, NumKeys, Seq), + bitcask:close(Ref), + Caller ! {fold_done, Seq}. + +fold(Ref, NumKeys, MinSeq) -> + Folder = fun(<>, Bin, {MinSeq2, Keys}) -> + {Key, Seq} = binary_to_term(Bin), + true = (Seq >= MinSeq2), + {MinSeq2, [Key | Keys]} + end, + {MinSeq, FoldedKeys} = bitcask:fold(Ref, Folder, {MinSeq, []}), + %% Exp = lists:seq(1, NumKeys), + %% Exp = lists:sort(FoldedKeys), % check we get the keys we expect + check_fold(1, NumKeys, lists:sort(FoldedKeys)). + +check_fold(Key, MaxKey, []) when Key == MaxKey + 1 -> + ok; +check_fold(Key, _MaxKey, []) -> + io:format("Fold missing key ~p (stopping searching)\n", [Key]); +check_fold(Key, MaxKey, [Key | Rest]) -> + check_fold(Key + 1, MaxKey, Rest); +check_fold(Key1, _MaxKey, [_Key2 | _Rest]) -> + io:format("Fold missing key ~p (stopping searching)\n", [Key1]). + + + +merge_proc(Cask, Caller) -> + ok = bitcask:merge(Cask), + Caller ! merge_done. + + +ensure_bitcask() -> + case code:ensure_loaded(bitcask) of + {module, bitcask} -> + ok; + _ -> + {ok, Cwd} = file:get_cwd(), + find_bitcask(filename:split(Cwd)) + end. + +%% Look for bitcask.beam in Cwd and Cwd/ebin +find_bitcask(["/"]) -> + erlang:error("Could not find bitcask\n"); +find_bitcask(Cwd) -> + case try_bitcask_dir(Cwd) of + true -> + ok; + false -> + case try_bitcask_dir(Cwd ++ ["ebin"]) of + true -> + ok; + false -> + find_bitcask(parent_dir(Cwd)) + end + end. + +try_bitcask_dir(Dir) -> + CodeDir = filename:join(Dir), + Beam = bitcask_beam(CodeDir), + io:format("Looking for bitcask in \"~s\".\n", [CodeDir]), + case filelib:is_regular(Beam) of + true -> + io:format("Adding bitcask dir \"~s\".\n", + [CodeDir]), + code:add_pathz(CodeDir), + {module, bitcask} = code:ensure_loaded(bitcask), + true; + _ -> + false + end. + +ensure_deps() -> + BitcaskBeam = code:where_is_file("bitcask.beam"), + BitcaskDir = parent_dir(filename:split(filename:dirname(BitcaskBeam))), + Pattern = filename:join(BitcaskDir) ++ "/deps/*/ebin", + Deps = filelib:wildcard(Pattern), + AddDepDir = fun(DepDir) -> + io:format("Adding dependency dir \"~s\".\n", + [DepDir]), + code:add_pathz(DepDir) + end, + lists:foreach(AddDepDir, Deps). + +parent_dir([]) -> + ["/"]; +parent_dir(["/"]) -> + ["/"]; +parent_dir(Dirs) -> + lists:reverse(tl(lists:reverse(Dirs))). + +bitcask_beam(Cwd) -> + filename:join(Cwd, ["bitcask" ++ code:objfile_extension()]). + +process_args([], State) -> + {ok, State}; +process_args([Arg | Rest], State) -> + case process_arg(Arg, State) of + {ok, NewState} -> + process_args(Rest, NewState); + Reason -> + Reason + end. + +process_arg({help, _}, _State) -> + syntax(), + syntax; +process_arg({Name, Val}, State) when Name =:= duration; + Name =:= status_freq; + Name =:= num_keys; + Name =:= readers; + Name =:= folders; + Name =:= mergers; + Name =:= max_file_size -> + case is_integer(Val) of + true -> + {ok, setelement(get_state_index(Name), State, Val)}; + false -> + {ok, setelement(get_state_index(Name), State, list_to_integer(Val))} + end; +process_arg({Name, Val}, State) when Name =:= restart_writer -> + case is_atom(Val) of + true -> + {ok, setelement(get_state_index(Name), State, Val)}; + false -> + {ok, setelement(get_state_index(Name), State, list_to_atom(Val))} + end; +process_arg({Name, Val}, State) -> + {ok, setelement(get_state_index(Name), State, Val)}. + +parse_arg([$- | Arg]) -> % strip leading --'s + parse_arg(Arg); +parse_arg(Arg) -> + case string:tokens(Arg, "=") of + [NameStr] -> + Val = undefined; + [NameStr, Val] -> + ok; + _ -> + NameStr = Val = undefined, + erlang:error({badarg, Arg}) + end, + case catch list_to_existing_atom(NameStr) of + {'EXIT', {badarg, _}} -> + Name = undefined, + erlang:error({badarg, NameStr}); + Name -> + ok + end, + {Name, Val}. + + +get_state_index(Name) -> + get_state_index(Name, 2, record_info(fields, state)). % first element is 'state' + +get_state_index(Name, _Index, []) -> + io:format("Cannot find index in state for ~p\n", [Name]), + erlang:error({badarg, Name}); +get_state_index(Name, Index, [Name | _Rest]) -> + Index; +get_state_index(Name, Index, [_OtherName | Rest]) -> + get_state_index(Name, Index + 1, Rest). + + +print_params(State) -> + io:format("Bitcask Test\n"), + io:format("duration: ~s\n", [format_duration(State#state.duration)]), + io:format("cask: ~s\n", [State#state.cask]), + io:format("num_keys: ~b\n", [State#state.num_keys]), + io:format("readers: ~b\n", [State#state.readers]), + io:format("folders: ~b\n", [State#state.folders]), + io:format("mergers: ~b\n", [State#state.mergers]), + io:format("max_file_size: ~s\n", [format_max_file_size(State#state.max_file_size)]), + io:format("\n"). + +format_duration(undefined) -> + "once"; +format_duration(Duration) -> + io_lib:format("~p ms", [Duration]). + +format_max_file_size(MaxFileSize) when is_integer(MaxFileSize), MaxFileSize > 0 -> + io_lib:format("~b", [MaxFileSize]); +format_max_file_size(_) -> + "default".