diff --git a/COPYING-GPLv2 b/COPYING-GPLv2 new file mode 100644 index 0000000..d159169 --- /dev/null +++ b/COPYING-GPLv2 @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/COPYING b/COPYING-LGPLv3 similarity index 100% rename from COPYING rename to COPYING-LGPLv3 diff --git a/GNUmakefile b/GNUmakefile index 4d3e2e5..226136d 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -1,12 +1,14 @@ # GNUmakefile -# +# # Copyright 2008 Bryan Ischo -# +# # This file is part of libs3. -# +# # libs3 is free software: you can redistribute it and/or modify it under the # terms of the GNU Lesser General Public License as published by the Free -# Software Foundation, version 3 of the License. +# Software Foundation, version 3 or above of the License. You can also +# redistribute and/or modify it under the terms of the GNU General Public +# License, version 2 or above of the License. # # In addition, as a special exception, the copyright holders give # permission to link the code of this library and its programs with the @@ -20,6 +22,10 @@ # You should have received a copy of the GNU Lesser General Public License # version 3 along with libs3, in a file named COPYING. If not, see # . +# +# You should also have received a copy of the GNU General Public License +# version 2 along with libs3, in a file named COPYING-GPLv2. If not, see +# . # I tried to use the autoconf/automake/autolocal/etc (i.e. autohell) tools # but I just couldn't stomach them. Since this is a Makefile for POSIX @@ -38,8 +44,8 @@ # -------------------------------------------------------------------------- # Set libs3 version number, unless it is already set. -LIBS3_VER_MAJOR ?= 2 -LIBS3_VER_MINOR ?= 0 +LIBS3_VER_MAJOR ?= 4 +LIBS3_VER_MINOR ?= 1 LIBS3_VER := $(LIBS3_VER_MAJOR).$(LIBS3_VER_MINOR) @@ -119,6 +125,9 @@ ifndef LIBXML2_CFLAGS LIBXML2_CFLAGS := $(shell xml2-config --cflags) endif +ifndef OPENSSL_LIBS + OPENSSL_LIBS := -lssl -lcrypto +endif # -------------------------------------------------------------------------- # These CFLAGS assume a GNU compiler. For other compilers, write a script @@ -133,7 +142,7 @@ ifndef CFLAGS endif endif -CFLAGS += -Wall -Werror -Wshadow -Wextra -Iinc \ +CFLAGS += -Wall -Wshadow -Wextra -Iinc \ $(CURL_CFLAGS) $(LIBXML2_CFLAGS) \ -DLIBS3_VER_MAJOR=\"$(LIBS3_VER_MAJOR)\" \ -DLIBS3_VER_MINOR=\"$(LIBS3_VER_MINOR)\" \ @@ -142,7 +151,10 @@ CFLAGS += -Wall -Werror -Wshadow -Wextra -Iinc \ -D_ISOC99_SOURCE \ -D_POSIX_C_SOURCE=200112L -LDFLAGS = $(CURL_LIBS) $(LIBXML2_LIBS) -lpthread +LDFLAGS = $(CURL_LIBS) $(LIBXML2_LIBS) $(OPENSSL_LIBS) -lpthread + +STRIP ?= strip +INSTALL := install --strip-program=$(STRIP) # -------------------------------------------------------------------------- @@ -156,7 +168,7 @@ all: exported test # Exported targets are the library and driver program .PHONY: exported -exported: libs3 s3 headers +exported: libs3 s3 s3-static headers # -------------------------------------------------------------------------- @@ -165,11 +177,11 @@ exported: libs3 s3 headers .PHONY: install install: exported $(QUIET_ECHO) $(DESTDIR)/bin/s3: Installing executable - $(VERBOSE_SHOW) install -Dps -m u+rwx,go+rx $(BUILD)/bin/s3 \ + $(VERBOSE_SHOW) $(INSTALL) -Dps -m u+rwx,go+rx $(BUILD)/bin/s3 \ $(DESTDIR)/bin/s3 $(QUIET_ECHO) \ $(LIBDIR)/libs3.so.$(LIBS3_VER): Installing shared library - $(VERBOSE_SHOW) install -Dps -m u+rw,go+r \ + $(VERBOSE_SHOW) $(INSTALL) -Dps -m u+rw,go+r \ $(BUILD)/lib/libs3.so.$(LIBS3_VER_MAJOR) \ $(LIBDIR)/libs3.so.$(LIBS3_VER) $(QUIET_ECHO) \ @@ -179,10 +191,10 @@ install: exported $(QUIET_ECHO) $(LIBDIR)/libs3.so: Linking shared library $(VERBOSE_SHOW) ln -sf libs3.so.$(LIBS3_VER_MAJOR) $(LIBDIR)/libs3.so $(QUIET_ECHO) $(LIBDIR)/libs3.a: Installing static library - $(VERBOSE_SHOW) install -Dp -m u+rw,go+r $(BUILD)/lib/libs3.a \ + $(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r $(BUILD)/lib/libs3.a \ $(LIBDIR)/libs3.a $(QUIET_ECHO) $(DESTDIR)/include/libs3.h: Installing header - $(VERBOSE_SHOW) install -Dp -m u+rw,go+r $(BUILD)/include/libs3.h \ + $(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r $(BUILD)/include/libs3.h \ $(DESTDIR)/include/libs3.h @@ -218,7 +230,7 @@ $(BUILD)/obj/%.do: src/%.c @ $(CC) $(CFLAGS) -M -MG -MQ $@ -DCOMPILINGDEPENDENCIES \ -o $(BUILD)/dep/$(<:%.c=%.dd) -c $< @ mkdir -p $(dir $@) - $(VERBOSE_SHOW) $(CC) $(CFLAGS) -fpic -fPIC -o $@ -c $< + $(VERBOSE_SHOW) $(CC) $(CFLAGS) -fpic -fPIC -o $@ -c $< # -------------------------------------------------------------------------- @@ -230,7 +242,7 @@ LIBS3_STATIC = $(BUILD)/lib/libs3.a .PHONY: libs3 libs3: $(LIBS3_SHARED) $(LIBS3_STATIC) -LIBS3_SOURCES := acl.c bucket.c error_parser.c general.c \ +LIBS3_SOURCES := bucket.c bucket_metadata.c error_parser.c general.c \ object.c request.c request_context.c \ response_headers_handler.c service_access_logging.c \ service.c simplexml.c util.c multipart.c @@ -252,12 +264,17 @@ $(LIBS3_STATIC): $(LIBS3_SOURCES:%.c=$(BUILD)/obj/%.o) .PHONY: s3 s3: $(BUILD)/bin/s3 +s3-static: $(BUILD)/bin/s3-static $(BUILD)/bin/s3: $(BUILD)/obj/s3.o $(LIBS3_SHARED) $(QUIET_ECHO) $@: Building executable @ mkdir -p $(dir $@) $(VERBOSE_SHOW) $(CC) -o $@ $^ $(LDFLAGS) +$(BUILD)/bin/s3-static: $(BUILD)/obj/s3.o $(LIBS3_STATIC) + $(QUIET_ECHO) $@: Building static executable + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(CC) -o $@ $^ $(LDFLAGS) # -------------------------------------------------------------------------- # libs3 header targets diff --git a/GNUmakefile.mingw b/GNUmakefile.mingw index f9ff2ba..bad5b78 100644 --- a/GNUmakefile.mingw +++ b/GNUmakefile.mingw @@ -1,12 +1,12 @@ # GNUmakefile.mingw -# -# Copyright 2008 Bryan Ischo -# +# # This file is part of libs3. -# +# # libs3 is free software: you can redistribute it and/or modify it under the # terms of the GNU Lesser General Public License as published by the Free -# Software Foundation, version 3 of the License. +# Software Foundation, version 3 or above of the License. You can also +# redistribute and/or modify it under the terms of the GNU General Public +# License, version 2 or above of the License. # # In addition, as a special exception, the copyright holders give # permission to link the code of this library and its programs with the @@ -20,6 +20,10 @@ # You should have received a copy of the GNU Lesser General Public License # version 3 along with libs3, in a file named COPYING. If not, see # . +# +# You should also have received a copy of the GNU General Public License +# version 2 along with libs3, in a file named COPYING-GPLv2. If not, see +# . # I tried to use the autoconf/automake/autolocal/etc (i.e. autohell) tools # but I just couldn't stomach them. Since this is a Makefile for POSIX @@ -38,8 +42,8 @@ # -------------------------------------------------------------------------- # Set libs3 version number, unless it is already set. -LIBS3_VER_MAJOR ?= 2 -LIBS3_VER_MINOR ?= 0 +LIBS3_VER_MAJOR ?= 4 +LIBS3_VER_MINOR ?= 1 LIBS3_VER := $(LIBS3_VER_MAJOR).$(LIBS3_VER_MINOR) @@ -212,7 +216,7 @@ LIBS3_STATIC = $(BUILD)/lib/libs3.a .PHONY: libs3 libs3: $(LIBS3_SHARED) $(BUILD)/lib/libs3.a -LIBS3_SOURCES := src/acl.c src/bucket.c src/error_parser.c src/general.c \ +LIBS3_SOURCES := src/bucket.c src/bucket_metadata.c src/error_parser.c src/general.c \ src/object.c src/request.c src/request_context.c \ src/response_headers_handler.c src/service_access_logging.c \ src/service.c src/simplexml.c src/util.c src/multipart.c \ diff --git a/GNUmakefile.osx b/GNUmakefile.osx index 76a45e9..c56d1d7 100644 --- a/GNUmakefile.osx +++ b/GNUmakefile.osx @@ -1,12 +1,12 @@ # GNUmakefile.osx -# -# Copyright 2008 Bryan Ischo -# +# # This file is part of libs3. -# +# # libs3 is free software: you can redistribute it and/or modify it under the # terms of the GNU Lesser General Public License as published by the Free -# Software Foundation, version 3 of the License. +# Software Foundation, version 3 or above of the License. You can also +# redistribute and/or modify it under the terms of the GNU General Public +# License, version 2 or above of the License. # # In addition, as a special exception, the copyright holders give # permission to link the code of this library and its programs with the @@ -20,6 +20,10 @@ # You should have received a copy of the GNU Lesser General Public License # version 3 along with libs3, in a file named COPYING. If not, see # . +# +# You should also have received a copy of the GNU General Public License +# version 2 along with libs3, in a file named COPYING-GPLv2. If not, see +# . # I tried to use the autoconf/automake/autolocal/etc (i.e. autohell) tools # but I just couldn't stomach them. Since this is a Makefile for POSIX @@ -38,8 +42,8 @@ # -------------------------------------------------------------------------- # Set libs3 version number, unless it is already set. -LIBS3_VER_MAJOR ?= 2 -LIBS3_VER_MINOR ?= 0 +LIBS3_VER_MAJOR ?= 4 +LIBS3_VER_MINOR ?= 1 LIBS3_VER := $(LIBS3_VER_MAJOR).$(LIBS3_VER_MINOR) @@ -102,6 +106,9 @@ endif ifndef LIBXML2_LIBS LIBXML2_LIBS := $(shell xml2-config --libs) + # Work around missing libsystem_symptoms.dylib in Xcode 8; see + # http://stackoverflow.com/questions/39536144/libsystem-symptoms-dylib-missing-in-xcode-8 + LIBXML2_LIBS := $(filter-out -L$(shell xcrun --show-sdk-path)/usr/lib, $(LIBXML2_LIBS)) endif ifndef LIBXML2_CFLAGS @@ -122,7 +129,12 @@ ifndef CFLAGS endif endif -CFLAGS += -Wall -Werror -Wshadow -Wextra -Iinc \ +# -------------------------------------------------------------------------- +# -Werror breaks the build on the macOS Sierra rendering build unusable +# so we use -Wunused-parameter flag instead that will only issue a warning +# with the newest clang compiler + +CFLAGS += -Wall -Wunused-parameter -Wshadow -Wextra -Iinc \ $(CURL_CFLAGS) $(LIBXML2_CFLAGS) \ -DLIBS3_VER_MAJOR=\"$(LIBS3_VER_MAJOR)\" \ -DLIBS3_VER_MINOR=\"$(LIBS3_VER_MINOR)\" \ @@ -208,7 +220,7 @@ $(BUILD)/obj/%.do: src/%.c @ gcc $(CFLAGS) -M -MG -MQ $@ -DCOMPILINGDEPENDENCIES \ -o $(BUILD)/dep/$(<:%.c=%.dd) -c $< @ mkdir -p $(dir $@) - $(VERBOSE_SHOW) gcc $(CFLAGS) -fpic -fPIC -o $@ -c $< + $(VERBOSE_SHOW) gcc $(CFLAGS) -fpic -fPIC -o $@ -c $< # -------------------------------------------------------------------------- @@ -220,7 +232,7 @@ LIBS3_STATIC = $(BUILD)/lib/libs3.a .PHONY: libs3 libs3: $(LIBS3_SHARED) $(LIBS3_SHARED_MAJOR) $(BUILD)/lib/libs3.a -LIBS3_SOURCES := src/acl.c src/bucket.c src/error_parser.c src/general.c \ +LIBS3_SOURCES := src/bucket.c src/bucket_metadata.c src/error_parser.c src/general.c \ src/object.c src/request.c src/request_context.c \ src/response_headers_handler.c src/service_access_logging.c \ src/service.c src/simplexml.c src/util.c src/multipart.c @@ -274,7 +286,6 @@ $(BUILD)/bin/testsimplexml: $(BUILD)/obj/testsimplexml.o $(LIBS3_STATIC) @ mkdir -p $(dir $@) $(VERBOSE_SHOW) gcc -o $@ $^ $(LIBXML2_LIBS) - # -------------------------------------------------------------------------- # Clean target diff --git a/LICENSE b/LICENSE index adcbfa5..ecd700e 100644 --- a/LICENSE +++ b/LICENSE @@ -2,7 +2,11 @@ Copyright 2008 Bryan Ischo libs3 is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free -Software Foundation, version 3 of the License. +Software Foundation, version 3 or above of the License. + +You can also redistribute it and/or modify it under the terms of the +GNU General Public License as published by the Free Software Foundation, +version 2 or above of the License. In addition, as a special exception, the copyright holders give permission to link the code of this library and its programs with the @@ -14,5 +18,9 @@ FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU Lesser General Public License -version 3 along with libs3, in a file named COPYING. If not, see +version 3 along with libs3, in a file named COPYING-LGPLv3. If not, see +. + +You should also have received a copy of the GNU General Public License +version 2 along with libs3, in a file named COPYING-GPLv2. If not, see . diff --git a/TODO b/TODO index 5c05749..8508f2b 100644 --- a/TODO +++ b/TODO @@ -81,11 +81,6 @@ This is Amazon's "user" management API; not part of S3 although IAM users can be - POST /?delete -- 4 hours -=== Object Expiration Support === - -- Handle the /?lifecycle property: 8 hours - - === MFA Authentication === (part of Bucket Policy) diff --git a/inc/error_parser.h b/inc/error_parser.h index 3f946ac..86b2889 100644 --- a/inc/error_parser.h +++ b/inc/error_parser.h @@ -2,12 +2,14 @@ * error_parser.h * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #ifndef ERROR_PARSER_H diff --git a/inc/libs3.h b/inc/libs3.h index 4e0aaad..3318ef2 100644 --- a/inc/libs3.h +++ b/inc/libs3.h @@ -2,12 +2,14 @@ * @file libs3.h * @details * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #ifndef LIBS3_H @@ -95,10 +101,10 @@ extern "C" { * user of S3, it is always possible that these maximums may be too low in * some rare circumstances. Bug reports should this unlikely situation occur * would be most appreciated. - * + * * Threading Rules * --------------- - * + * * 1. All arguments passed to any function must not be modified directly until * the function returns. * 2. All S3RequestContext and S3Request arguments passed to all functions may @@ -207,7 +213,7 @@ extern "C" { /** * This constant is used by the S3_initialize() function, to specify that - * the winsock library should be initialized by libs3; only relevent on + * the winsock library should be initialized by libs3; only relevent on * Microsoft Windows platforms. **/ #define S3_INIT_WINSOCK 1 @@ -228,6 +234,12 @@ extern "C" { #define S3_INIT_ALL (S3_INIT_WINSOCK) +/** + * The default region identifier used to scope the signing key + */ +#define S3_DEFAULT_REGION "us-east-1" + + /** ************************************************************************** * Enumerations ************************************************************************** **/ @@ -292,7 +304,8 @@ typedef enum S3StatusServerFailedVerification , S3StatusConnectionFailed , S3StatusAbortedByCallback , - + S3StatusNotSupported , + /** * Errors from the S3 service **/ @@ -319,6 +332,7 @@ typedef enum S3StatusErrorInvalidBucketName , S3StatusErrorInvalidBucketState , S3StatusErrorInvalidDigest , + S3StatusErrorInvalidEncryptionAlgorithmError , S3StatusErrorInvalidLocationConstraint , S3StatusErrorInvalidObjectState , S3StatusErrorInvalidPart , @@ -354,7 +368,7 @@ typedef enum S3StatusErrorNoSuchVersion , S3StatusErrorNotImplemented , S3StatusErrorNotSignedUp , - S3StatusErrorNotSuchBucketPolicy , + S3StatusErrorNoSuchBucketPolicy , S3StatusErrorOperationAborted , S3StatusErrorPermanentRedirect , S3StatusErrorPreconditionFailed , @@ -373,6 +387,7 @@ typedef enum S3StatusErrorUnexpectedContent , S3StatusErrorUnresolvableGrantByEmailAddress , S3StatusErrorUserKeyMustBeSpecified , + S3StatusErrorQuotaExceeded , S3StatusErrorUnknown , /** @@ -478,7 +493,7 @@ typedef enum * Private canned ACL gives the owner FULL_CONTROL and no other permissions * are issued * Public Read canned ACL gives the owner FULL_CONTROL and all users Read - * permission + * permission * Public Read Write canned ACL gives the owner FULL_CONTROL and all users * Read and Write permission * AuthenticatedRead canned ACL gives the owner FULL_CONTROL and authenticated @@ -489,7 +504,8 @@ typedef enum S3CannedAclPrivate = 0, /* private */ S3CannedAclPublicRead = 1, /* public-read */ S3CannedAclPublicReadWrite = 2, /* public-read-write */ - S3CannedAclAuthenticatedRead = 3 /* authenticated-read */ + S3CannedAclAuthenticatedRead = 3, /* authenticated-read */ + S3CannedAclBucketOwnerFullControl = 4 /* bucket-owner-full-control */ } S3CannedAcl; @@ -577,7 +593,7 @@ typedef struct S3ResponseProperties * modified time was not provided in the response. If this value is >= 0, * then the last modified date of the contents are available as a number * of seconds since the UNIX epoch. - * + * **/ int64_t lastModified; @@ -633,7 +649,7 @@ typedef struct S3AclGrant union { /** - * This structure is used iff the granteeType is + * This structure is used iff the granteeType is * S3GranteeTypeAmazonCustomerByEmail. **/ struct @@ -711,6 +727,12 @@ typedef struct S3BucketContext * The Amazon Security Token used to generate Temporary Security Credentials **/ const char *securityToken; + + /** + * The AWS region to which to scope the signing key used for authorization. + * If NULL, the default region ("us-east-1") will be used. + */ + const char *authRegion; } S3BucketContext; @@ -728,7 +750,7 @@ typedef struct S3ListBucketContent /** * This is the number of seconds since UNIX epoch of the last modified - * date of the object identified by the key. + * date of the object identified by the key. **/ int64_t lastModified; @@ -784,25 +806,25 @@ typedef struct S3ListMultipartUpload * access permissions allow it to be viewed. **/ const char *ownerDisplayName; - + const char *storageClass; /** * This is the number of seconds since UNIX epoch of the last modified - * date of the object identified by the key. + * date of the object identified by the key. **/ int64_t initiated; - + } S3ListMultipartUpload; typedef struct S3ListPart -{ +{ const char *eTag; /** * This is the number of seconds since UNIX epoch of the last modified - * date of the object identified by the key. + * date of the object identified by the key. **/ int64_t lastModified; uint64_t partNumber; @@ -1014,7 +1036,7 @@ typedef void (S3ResponseCompleteCallback)(S3Status status, const S3ErrorDetails *errorDetails, void *callbackData); - + /** * This callback is made for each bucket resulting from a list service * operation. @@ -1033,7 +1055,7 @@ typedef void (S3ResponseCompleteCallback)(S3Status status, * Typically, this will return either S3StatusOK or * S3StatusAbortedByCallback. **/ -typedef S3Status (S3ListServiceCallback)(const char *ownerId, +typedef S3Status (S3ListServiceCallback)(const char *ownerId, const char *ownerDisplayName, const char *bucketName, int64_t creationDateSeconds, @@ -1072,12 +1094,12 @@ typedef S3Status (S3ListServiceCallback)(const char *ownerId, **/ typedef S3Status (S3ListBucketCallback)(int isTruncated, const char *nextMarker, - int contentsCount, + int contentsCount, const S3ListBucketContent *contents, int commonPrefixesCount, const char **commonPrefixes, void *callbackData); - + /** * This callback is made during a put object operation, to obtain the next @@ -1121,7 +1143,7 @@ typedef int (S3PutObjectDataCallback)(int bufferSize, char *buffer, **/ typedef S3Status (S3GetObjectDataCallback)(int bufferSize, const char *buffer, void *callbackData); - + /** * This callback is made after initiation of a multipart upload operation. It @@ -1169,7 +1191,7 @@ typedef S3Status (S3MultipartInitialResponseCallback)(const char *upload_id, **/ typedef S3Status (S3ListMultipartUploadsResponseCallback) (int isTruncated, const char *nextKeyMarker, - const char *nextUploadIdMarker, int uploadsCount, + const char *nextUploadIdMarker, int uploadsCount, const S3ListMultipartUpload *uploads, int commonPrefixesCount, const char **commonPrefixes, void *callbackData); @@ -1225,6 +1247,28 @@ typedef S3Status (S3MultipartCommitResponseCallback)(const char *location, void *callbackData); +/** + * Mechanism for S3 application to customize each CURL easy request + * associated with the given S3 request context. + * + * This callback can be optinally configured using S3_create_request_context_ex + * and will be invoked every time a new CURL request is created in the + * context of the given CURLM handle. Invocation will occur after + * libs3 has finished configuring its own options of CURL, but before + * CURL is started. + * + * @param curl_multi is the CURLM handle associated with this context. + * @param curl_easy is the CURL request being created. + * @param setupData is the setupCurlCallbackData parameter passed to + * S3_create_request_context_ex. + * @return S3StatusOK to continue processing the request, anything else to + * immediately abort the request and pass this status + * to the S3ResponseCompleteCallback for this request. + **/ +typedef S3Status (*S3SetupCurlCallback)(void *curlMulti, void *curlEasy, + void *setupData); + + /** ************************************************************************** * Callback Structures ************************************************************************** **/ @@ -1242,7 +1286,7 @@ typedef struct S3ResponseHandler * if the response properties were not successfully returned from S3. **/ S3ResponsePropertiesCallback *propertiesCallback; - + /** * The completeCallback is always called for every request made to S3, * regardless of the outcome of the request. It provides the status of @@ -1368,7 +1412,7 @@ typedef struct S3ListMultipartUploadsHandler * responseHandler provides the properties and complete callback **/ S3ResponseHandler responseHandler; - + S3ListMultipartUploadsResponseCallback *responseXmlCallback; } S3ListMultipartUploadsHandler; @@ -1378,7 +1422,7 @@ typedef struct S3ListPartsHandler * responseHandler provides the properties and complete callback **/ S3ResponseHandler responseHandler; - + S3ListPartsResponseCallback *responseXmlCallback; } S3ListPartsHandler; @@ -1388,7 +1432,7 @@ typedef struct S3AbortMultipartUploadHandler * responseHandler provides the properties and complete callback **/ S3ResponseHandler responseHandler; - + } S3AbortMultipartUploadHandler; /** ************************************************************************** @@ -1528,7 +1572,7 @@ S3Status S3_validate_bucket_name(const char *bucketName, S3UriStyle uriStyle); **/ S3Status S3_convert_acl(char *aclXml, char *ownerId, char *ownerDisplayName, int *aclGrantCountReturn, S3AclGrant *aclGrants); - + /** * Returns nonzero if the status indicates that the request should be @@ -1568,6 +1612,51 @@ int S3_status_is_retryable(S3Status status); S3Status S3_create_request_context(S3RequestContext **requestContextReturn); +/** + * Extended version of S3_create_request_context used to create S3RequestContext + * for curl_multi_socket_action CURLM handles that will be managed by libs3 user. + * This type of handles offer better performance for applications with large + * number of simultaneous connections. For details, see MULTI_SOCKET chapter here: + * https://curl.haxx.se/libcurl/c/libcurl-multi.html + * + * In this mode libs3 user will + * - create its own CURLM using curl_multi_init() + * - configure it for its own handlers using + * CURLMOPT_SOCKETFUNCTION/CURLMOPT_TIMERFUNCTION/etc + * - use S3_create_request_context_ex to create S3RequestContext + * for the above CURLM handle + * - start S3 request + * - every time setupCurlCallback is called, will configure new CURL + * object with its own handlers using + * CURLOPT_OPENSOCKETFUNCTION/CURLOPT_CLOSESOCKETFUNCTION/etc + * - the moment libs3 adds CURL object to CURLM handle, curl will start + * communicating directly with libs3 user to drive socket operations, + * where libs3 user will be responsible for calling curl_multi_socket_action + * when necessary. + * - whenever curl_multi_socket_action indicates change in running_handles + * libs3 user should call S3_process_request_context to let libs3 process + * any completed curl transfers and notify back to libs3 user if that was + * the final transfer for a given S3 request. + * + * @param requestContextReturn returns the newly-created S3RequestContext + * structure, which if successfully returned, must be destroyed via a + * call to S3_destroy_request_context when it is no longer needed. If + * an error status is returned from this function, then + * requestContextReturn will not have been filled in, and + * S3_destroy_request_context should not be called on it + * @param curlMulti is the CURLM handle to be associated with this context. + * @param setupCurlCallback is an optional callback routine to be invoked + * by libs3 every time another CURL request is being created for + * use in this context. + * @param setupCurlCallbackData is an opaque data to be passed to + * setupCurlCallback. + **/ +S3Status S3_create_request_context_ex(S3RequestContext **requestContextReturn, + void *curlMulti, + S3SetupCurlCallback setupCurlCallback, + void *setupCurlCallbackData); + + /** * Destroys an S3RequestContext which was created with * S3_create_request_context. Any requests which are currently being @@ -1621,10 +1710,30 @@ S3Status S3_runall_request_context(S3RequestContext *requestContext); * S3StatusOutOfMemory if requests could not be processed due to * an out of memory error **/ -S3Status S3_runonce_request_context(S3RequestContext *requestContext, +S3Status S3_runonce_request_context(S3RequestContext *requestContext, int *requestsRemainingReturn); +/** + * Extract and finish requests completed by curl multi handle mechanism + * in curl_multi_socket_action mode. Should be called by libs3 user when + * curl_multi_socket_action indicates a change in running_handles. + * + * @param requestContext is the S3RequestContext to process + * @return One of: + * S3StatusOK if request processing proceeded without error + * S3StatusConnectionFailed if the socket connection to the server + * failed + * S3StatusServerFailedVerification if the SSL certificate of the + * server could not be verified. + * S3StatusInternalError if an internal error prevented the + * S3RequestContext from running one or more requests + * S3StatusOutOfMemory if requests could not be processed due to + * an out of memory error + **/ +S3Status S3_process_request_context(S3RequestContext *requestContext); + + /** * This function, in conjunction allows callers to manually manage a set of * requests using an S3RequestContext. This function returns the set of file @@ -1701,7 +1810,7 @@ void S3_set_request_context_verify_peer(S3RequestContext *requestContext, * of authenticated query string request. * * @param buffer is the output buffer for the authenticated query string. - * It must be at least S3_MAX_AUTHENTICATED_QUERY_STRING_SIZE bytes in + * It must be at least S3_MAX_AUTHENTICATED_QUERY_STRING_SIZE bytes in * length. * @param bucketContext gives the bucket and associated parameters for the * request to generate. @@ -1709,10 +1818,12 @@ void S3_set_request_context_verify_peer(S3RequestContext *requestContext, * @param expires gives the number of seconds since Unix epoch for the * expiration date of the request; after this time, the request will * no longer be valid. If this value is negative, the largest - * expiration date possible is used (currently, Jan 19, 2038). + * expiration interval possible is used (one week). * @param resource gives a sub-resource to be fetched for the request, or NULL - * for none. This should be of the form "?", i.e. + * for none. This should be of the form "?", i.e. * "?torrent". + * @param httpMethod the HTTP request method that will be used with the + * generated query string (e.g. "GET"). * @return One of: * S3StatusUriTooLong if, due to an internal error, the generated URI * is longer than S3_MAX_AUTHENTICATED_QUERY_STRING_SIZE bytes in @@ -1721,7 +1832,8 @@ void S3_set_request_context_verify_peer(S3RequestContext *requestContext, **/ S3Status S3_generate_authenticated_query_string (char *buffer, const S3BucketContext *bucketContext, - const char *key, int64_t expires, const char *resource); + const char *key, int expires, const char *resource, + const char *httpMethod); /** ************************************************************************** @@ -1736,24 +1848,27 @@ S3Status S3_generate_authenticated_query_string * buckets * @param secretAccessKey gives the Amazon Secret Access Key for which to list * owned buckets - * @param securityToken gives the security token used to generate the Temporary + * @param securityToken gives the security token used to generate the Temporary * Security Credentials * @param hostName is the S3 host name to use; if NULL is passed in, the * default S3 host as provided to S3_initialize() will be used. + * @param authRegion is the AWS region to use for the authorization signature * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ void S3_list_service(S3Protocol protocol, const char *accessKeyId, const char *secretAccessKey, const char *securityToken, - const char *hostName, S3RequestContext *requestContext, - const S3ListServiceHandler *handler, - void *callbackData); - + const char *hostName, const char *authRegion, + S3RequestContext *requestContext, + int timeoutMs, + const S3ListServiceHandler *handler, void *callbackData); + /** ************************************************************************** * Bucket Functions @@ -1768,11 +1883,12 @@ void S3_list_service(S3Protocol protocol, const char *accessKeyId, * buckets * @param secretAccessKey gives the Amazon Secret Access Key for which to list * owned buckets - * @param securityToken gives the security token used to generate the Temporary + * @param securityToken gives the security token used to generate the Temporary * Security Credentials * @param hostName is the S3 host name to use; if NULL is passed in, the * default S3 host as provided to S3_initialize() will be used. * @param bucketName is the bucket name to test + * @param authRegion is the AWS region to use for the authorization signature * @param locationConstraintReturnSize gives the number of bytes in the * locationConstraintReturn parameter * @param locationConstraintReturn provides the location into which to write @@ -1785,20 +1901,23 @@ void S3_list_service(S3Protocol protocol, const char *accessKeyId, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle, const char *accessKeyId, const char *secretAccessKey, - const char *securityToken, const char *hostName, - const char *bucketName, int locationConstraintReturnSize, + const char *securityToken, const char *hostName, + const char *bucketName, const char *authRegion, + int locationConstraintReturnSize, char *locationConstraintReturn, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData); - + /** * Creates a new bucket. * @@ -1807,27 +1926,31 @@ void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle, * buckets * @param secretAccessKey gives the Amazon Secret Access Key for which to list * owned buckets - * @param securityToken gives the security token used to generate the Temporary + * @param securityToken gives the security token used to generate the Temporary * Security Credentials * @param hostName is the S3 host name to use; if NULL is passed in, the * default S3 host as provided to S3_initialize() will be used. * @param bucketName is the name of the bucket to be created + * @param authRegion is the AWS region to use for the authorization signature * @param cannedAcl gives the "REST canned ACL" to use for the created bucket * @param locationConstraint if non-NULL, gives the geographic location for * the bucket to create. * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ void S3_create_bucket(S3Protocol protocol, const char *accessKeyId, const char *secretAccessKey, const char *securityToken, const char *hostName, const char *bucketName, - S3CannedAcl cannedAcl, const char *locationConstraint, + const char *authRegion, S3CannedAcl cannedAcl, + const char *locationConstraint, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData); @@ -1841,23 +1964,27 @@ void S3_create_bucket(S3Protocol protocol, const char *accessKeyId, * buckets * @param secretAccessKey gives the Amazon Secret Access Key for which to list * owned buckets - * @param securityToken gives the security token used to generate the Temporary + * @param securityToken gives the security token used to generate the Temporary * Security Credentials * @param hostName is the S3 host name to use; if NULL is passed in, the * default S3 host as provided to S3_initialize() will be used. * @param bucketName is the name of the bucket to be deleted + * @param authRegion is the AWS region to use for the authorization signature * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ void S3_delete_bucket(S3Protocol protocol, S3UriStyle uriStyle, const char *accessKeyId, const char *secretAccessKey, - const char *securityToken, const char *hostName, - const char *bucketName, S3RequestContext *requestContext, + const char *securityToken, const char *hostName, + const char *bucketName, const char *authRegion, + S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData); @@ -1876,15 +2003,17 @@ void S3_delete_bucket(S3Protocol protocol, S3UriStyle uriStyle, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ void S3_list_bucket(const S3BucketContext *bucketContext, - const char *prefix, const char *marker, + const char *prefix, const char *marker, const char *delimiter, int maxkeys, S3RequestContext *requestContext, + int timeoutMs, const S3ListBucketHandler *handler, void *callbackData); @@ -1907,8 +2036,9 @@ void S3_list_bucket(const S3BucketContext *bucketContext, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ @@ -1916,8 +2046,9 @@ void S3_put_object(const S3BucketContext *bucketContext, const char *key, uint64_t contentLength, const S3PutProperties *putProperties, S3RequestContext *requestContext, + int timeoutMs, const S3PutObjectHandler *handler, void *callbackData); - + /** * Copies an object from one location to another. The object may be copied @@ -1942,14 +2073,15 @@ void S3_put_object(const S3BucketContext *bucketContext, const char *key, * @param eTagReturn is a buffer into which the resulting eTag of the copied * object will be written * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ @@ -1959,6 +2091,7 @@ void S3_copy_object(const S3BucketContext *bucketContext, const S3PutProperties *putProperties, int64_t *lastModifiedReturn, int eTagReturnSize, char *eTagReturn, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData); @@ -1997,6 +2130,7 @@ void S3_copy_object(const S3BucketContext *bucketContext, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and * completed * @param callbackData will be passed in as the callbackData parameter to @@ -2010,6 +2144,7 @@ void S3_copy_object_range(const S3BucketContext *bucketContext, const S3PutProperties *putProperties, int64_t *lastModifiedReturn, int eTagReturnSize, char *eTagReturn, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData); @@ -2029,8 +2164,9 @@ void S3_copy_object_range(const S3BucketContext *bucketContext, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ @@ -2038,6 +2174,7 @@ void S3_get_object(const S3BucketContext *bucketContext, const char *key, const S3GetConditions *getConditions, uint64_t startByte, uint64_t byteCount, S3RequestContext *requestContext, + int timeoutMs, const S3GetObjectHandler *handler, void *callbackData); @@ -2050,15 +2187,17 @@ void S3_get_object(const S3BucketContext *bucketContext, const char *key, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ void S3_head_object(const S3BucketContext *bucketContext, const char *key, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData); - + /** * Deletes an object from S3. * @@ -2068,13 +2207,15 @@ void S3_head_object(const S3BucketContext *bucketContext, const char *key, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ void S3_delete_object(const S3BucketContext *bucketContext, const char *key, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData); @@ -2103,15 +2244,17 @@ void S3_delete_object(const S3BucketContext *bucketContext, const char *key, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ -void S3_get_acl(const S3BucketContext *bucketContext, const char *key, +void S3_get_acl(const S3BucketContext *bucketContext, const char *key, char *ownerId, char *ownerDisplayName, - int *aclGrantCountReturn, S3AclGrant *aclGrants, + int *aclGrantCountReturn, S3AclGrant *aclGrants, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData); @@ -2135,18 +2278,68 @@ void S3_get_acl(const S3BucketContext *bucketContext, const char *key, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ -void S3_set_acl(const S3BucketContext *bucketContext, const char *key, +void S3_set_acl(const S3BucketContext *bucketContext, const char *key, const char *ownerId, const char *ownerDisplayName, - int aclGrantCount, const S3AclGrant *aclGrants, + int aclGrantCount, const S3AclGrant *aclGrants, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData); +/** ************************************************************************** + * Lifecycle Control Functions + ************************************************************************** **/ + +/** + * Gets the lifecycle for the given bucket + * + * @param bucketContext gives the bucket and associated parameters for this + * request + * @param lifecycleXmlDocumentReturn buffer for lifecycle XML document + * @param lifecycleXmlDocumentBufferSize size of the buffer + * @param requestContext if non-NULL, gives the S3RequestContext to add this + * request to, and does not perform the request immediately. If NULL, + * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds + * @param handler gives the callbacks to call as the request is processed and + * completed + * @param callbackData will be passed in as the callbackData parameter to + * all callbacks for this request + **/ +void S3_get_lifecycle(const S3BucketContext *bucketContext, + char *lifecycleXmlDocumentReturn, int lifecycleXmlDocumentBufferSize, + S3RequestContext *requestContext, + int timeoutMs, + const S3ResponseHandler *handler, void *callbackData); + + +/** + * Sets the lifecycle for the given bucket + * + * @param bucketContext gives the bucket and associated parameters for this + * request + * @param lifecycleXmlDocument Lifecycle configuration as an XML document + * @param requestContext if non-NULL, gives the S3RequestContext to add this + * request to, and does not perform the request immediately. If NULL, + * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds + * @param handler gives the callbacks to call as the request is processed and + * completed + * @param callbackData will be passed in as the callbackData parameter to + * all callbacks for this request + **/ +void S3_set_lifecycle(const S3BucketContext *bucketContext, + const char *lifecycleXmlDocument, + S3RequestContext *requestContext, + int timeoutMs, + const S3ResponseHandler *handler, void *callbackData); + /** ************************************************************************** * Server Access Log Functions ************************************************************************** **/ @@ -2182,20 +2375,22 @@ void S3_set_acl(const S3BucketContext *bucketContext, const char *key, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ void S3_get_server_access_logging(const S3BucketContext *bucketContext, char *targetBucketReturn, char *targetPrefixReturn, - int *aclGrantCountReturn, + int *aclGrantCountReturn, S3AclGrant *aclGrants, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData); - + /** * Sets the service access logging settings for a bucket. The service access @@ -2221,24 +2416,26 @@ void S3_get_server_access_logging(const S3BucketContext *bucketContext, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ void S3_set_server_access_logging(const S3BucketContext *bucketContext, - const char *targetBucket, - const char *targetPrefix, int aclGrantCount, - const S3AclGrant *aclGrants, + const char *targetBucket, + const char *targetPrefix, int aclGrantCount, + const S3AclGrant *aclGrants, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData); - + /** - * This operation initiates a multipart upload and returns an upload ID. - * This upload ID is used to associate all the parts in the specific - * multipart upload. You specify this upload ID in each of your subsequent + * This operation initiates a multipart upload and returns an upload ID. + * This upload ID is used to associate all the parts in the specific + * multipart upload. You specify this upload ID in each of your subsequent * upload part requests * * @param bucketContext gives the bucket and associated parameters for this @@ -2248,10 +2445,11 @@ void S3_set_server_access_logging(const S3BucketContext *bucketContext, * @param putProperties optionally provides additional properties to apply to * the object that is being put to * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ @@ -2259,6 +2457,7 @@ void S3_initiate_multipart(S3BucketContext *bucketContext, const char *key, S3PutProperties *putProperties, S3MultipartInitialHandler *handler, S3RequestContext *requestContext, + int timeoutMs, void *callbackData); @@ -2272,14 +2471,15 @@ void S3_initiate_multipart(S3BucketContext *bucketContext, const char *key, * @param putProperties optionally provides additional properties to apply to * the object that is being put to * @param handler gives the callbacks to call as the request is processed and - * completed - * @param seq is a part number uniquely identifies a part and also - * defines its position within the object being created. - * @param upload_id get from S3_initiate_multipart return - * @param partContentLength gives the size of the part, in bytes + * completed + * @param seq is a part number uniquely identifies a part and also + * defines its position within the object being created. + * @param upload_id get from S3_initiate_multipart return + * @param partContentLength gives the size of the part, in bytes * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ @@ -2287,11 +2487,13 @@ void S3_upload_part(S3BucketContext *bucketContext, const char *key, S3PutProperties * putProperties, S3PutObjectHandler *handler, int seq, const char *upload_id, int partContentLength, - S3RequestContext *requestContext, void *callbackData); + S3RequestContext *requestContext, + int timeoutMs, + void *callbackData); /** - * This operation completes a multipart upload by assembling previously + * This operation completes a multipart upload by assembling previously * uploaded parts. * * @param bucketContext gives the bucket and associated parameters for this @@ -2300,25 +2502,27 @@ void S3_upload_part(S3BucketContext *bucketContext, const char *key, * @param key is the source key * @param handler gives the callbacks to call as the request is processed and * completed - * @param upload_id get from S3_initiate_multipart return - * @param contentLength gives the total size of the commit message, in bytes + * @param upload_id get from S3_initiate_multipart return + * @param contentLength gives the total size of the commit message, in bytes * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ -void S3_complete_multipart_upload(S3BucketContext *bucketContext, +void S3_complete_multipart_upload(S3BucketContext *bucketContext, const char *key, S3MultipartCommitHandler *handler, const char *upload_id, int contentLength, S3RequestContext *requestContext, + int timeoutMs, void *callbackData); /** - * This operation lists the parts that have been uploaded for a specific + * This operation lists the parts that have been uploaded for a specific * multipart upload. * * @param bucketContext gives the bucket and associated parameters for this @@ -2329,14 +2533,15 @@ void S3_complete_multipart_upload(S3BucketContext *bucketContext, * which listing should begin. Only parts with higher part numbers * will be listed. * @param uploadid identifying the multipart upload whose parts are being - * listed. + * listed. * @param encodingtype if present and non-empty, requests Amazon S3 to encode * the response and specifies the encoding method to use. - * @param maxparts Sets the maximum number of parts to return in the response + * @param maxparts Sets the maximum number of parts to return in the response * body. Default: 1,000 * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and * completed * @param callbackData will be passed in as the callbackData parameter to @@ -2344,33 +2549,36 @@ void S3_complete_multipart_upload(S3BucketContext *bucketContext, **/ void S3_list_parts(S3BucketContext *bucketContext, const char *key, const char *partnumbermarker, - const char *uploadid, const char *encodingtype, + const char *uploadid, const char *encodingtype, int maxparts, S3RequestContext *requestContext, + int timeoutMs, const S3ListPartsHandler *handler, void *callbackData); /** - * This operation aborts a multipart upload. After a multipart upload is - * aborted, no additional parts can be uploaded using that upload ID. + * This operation aborts a multipart upload. After a multipart upload is + * aborted, no additional parts can be uploaded using that upload ID. * * @param bucketContext gives the bucket and associated parameters for this * request; this is the bucket for which service access logging is * being set * @param key is the source key * @param uploadId identifying the multipart upload whose parts are being - * listed. + * listed. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and * completed **/ void S3_abort_multipart_upload(S3BucketContext *bucketContext, const char *key, const char *uploadId, + int timeoutMs, S3AbortMultipartUploadHandler *handler); /** - * This operation lists in-progress multipart uploads. An in-progress - * multipart upload is a multipart upload that has been initiated, - * using the Initiate Multipart Upload request, but has not yet been + * This operation lists in-progress multipart uploads. An in-progress + * multipart upload is a multipart upload that has been initiated, + * using the Initiate Multipart Upload request, but has not yet been * completed or aborted. * * @param bucketContext gives the bucket and associated parameters for this @@ -2386,22 +2594,24 @@ void S3_abort_multipart_upload(S3BucketContext *bucketContext, const char *key, * @param encodingtype if present and non-empty, requests Amazon S3 to encode * the response and specifies the encoding method to use. * @param delimiter if present and non-empty, is the character you use to - * group keys. - * @param maxuploads sets the maximum number of multipart uploads, - * from 1 to 1,000, to return in the response body. + * group keys. + * @param maxuploads sets the maximum number of multipart uploads, + * from 1 to 1,000, to return in the response body. * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request - **/ -void S3_list_multipart_uploads(S3BucketContext *bucketContext, + **/ +void S3_list_multipart_uploads(S3BucketContext *bucketContext, const char *prefix, const char *keymarker, const char *uploadidmarker, - const char *encodingtype, const char *delimiter, + const char *encodingtype, const char *delimiter, int maxuploads, S3RequestContext *requestContext, + int timeoutMs, const S3ListMultipartUploadsHandler *handler, void *callbackData); diff --git a/inc/mingw/pthread.h b/inc/mingw/pthread.h index 9664dd0..e87b580 100644 --- a/inc/mingw/pthread.h +++ b/inc/mingw/pthread.h @@ -2,12 +2,14 @@ * pthread.h * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #ifndef PTHREAD_H diff --git a/inc/mingw/sys/select.h b/inc/mingw/sys/select.h index c0b0484..3093717 100644 --- a/inc/mingw/sys/select.h +++ b/inc/mingw/sys/select.h @@ -2,12 +2,14 @@ * select.h * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ // This file is used only on a MingW build, and converts an include of diff --git a/inc/mingw/sys/utsname.h b/inc/mingw/sys/utsname.h index 8c351ee..44c4d22 100644 --- a/inc/mingw/sys/utsname.h +++ b/inc/mingw/sys/utsname.h @@ -2,12 +2,14 @@ * utsname.h * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ // This file is used only on a MingW build, and provides an implementation diff --git a/inc/request.h b/inc/request.h index b54e945..a80971a 100644 --- a/inc/request.h +++ b/inc/request.h @@ -1,13 +1,15 @@ /** ************************************************************************** * request.h - * + * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #ifndef REQUEST_H @@ -40,7 +46,8 @@ typedef enum HttpRequestTypePUT, HttpRequestTypeCOPY, HttpRequestTypeDELETE, - HttpRequestTypePOST + HttpRequestTypePOST, + HttpRequestTypeInvalid } HttpRequestType; @@ -74,10 +81,10 @@ typedef struct RequestParams const S3GetConditions *getConditions; // Start byte - uint64_t startByte; + size_t startByte; // Byte count - uint64_t byteCount; + size_t byteCount; // Put properties const S3PutProperties *putProperties; @@ -101,6 +108,9 @@ typedef struct RequestParams // Data passed to the callbacks void *callbackData; + + // Request timeout. If 0, no timeout will be enforced + int timeoutMs; } RequestParams; diff --git a/inc/request_context.h b/inc/request_context.h index 229ef35..e91d9c8 100644 --- a/inc/request_context.h +++ b/inc/request_context.h @@ -2,12 +2,14 @@ * request_context.h * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #ifndef REQUEST_CONTEXT_H @@ -29,14 +35,26 @@ #include "libs3.h" + +typedef enum +{ + S3CurlModeMultiPerform , + S3CurlModeMultiSocket , +} S3CurlMode; + + struct S3RequestContext { CURLM *curlm; + S3CurlMode curl_mode; int verifyPeerSet; long verifyPeer; struct Request *requests; + + S3SetupCurlCallback setupCurlCallback; + void *setupCurlCallbackData; }; diff --git a/inc/response_headers_handler.h b/inc/response_headers_handler.h index ba573e0..b2341c2 100644 --- a/inc/response_headers_handler.h +++ b/inc/response_headers_handler.h @@ -2,12 +2,14 @@ * response_headers_handler.h * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #ifndef RESPONSE_HEADERS_HANDLER_H diff --git a/inc/simplexml.h b/inc/simplexml.h index 4fbbf68..f12a22b 100644 --- a/inc/simplexml.h +++ b/inc/simplexml.h @@ -2,12 +2,14 @@ * simplexml.h * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #ifndef SIMPLEXML_H diff --git a/inc/string_buffer.h b/inc/string_buffer.h index d1cf5c3..a4968ad 100644 --- a/inc/string_buffer.h +++ b/inc/string_buffer.h @@ -2,12 +2,14 @@ * string_buffer.h * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #ifndef STRING_BUFFER_H diff --git a/inc/util.h b/inc/util.h index 854b9d9..8c66a55 100644 --- a/inc/util.h +++ b/inc/util.h @@ -2,12 +2,14 @@ * util.h * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #ifndef UTIL_H @@ -40,7 +46,6 @@ #define ACS_GROUP_LOG_DELIVERY ACS_URL "s3/LogDelivery" - // Derived from S3 documentation // This is the maximum number of bytes needed in a "compacted meta header" @@ -64,6 +69,12 @@ #define MAX_CANONICALIZED_RESOURCE_SIZE \ (1 + 255 + 1 + MAX_URLENCODED_KEY_SIZE + (sizeof("?torrent") - 1) + 1) +#define MAX_ACCESS_KEY_ID_LENGTH 32 + +// Maximum length of a credential string +// ///s3/aws4_request +#define MAX_CREDENTIAL_SIZE \ + (MAX_ACCESS_KEY_ID_LENGTH + 1) + 8 + 1 + 32 + sizeof("/s3/aws4_request") // Utilities ----------------------------------------------------------------- @@ -71,26 +82,13 @@ // 3x the number of characters that [source] has. At most [maxSrcSize] bytes // from [src] are encoded; if more are present in [src], 0 is returned from // urlEncode, else nonzero is returned. -int urlEncode(char *dest, const char *src, int maxSrcSize); +int urlEncode(char *dest, const char *src, int maxSrcSize, int encodeSlash); // Returns < 0 on failure >= 0 on success int64_t parseIso8601Time(const char *str); uint64_t parseUnsignedInt(const char *str); -// base64 encode bytes. The output buffer must have at least -// ((4 * (inLen + 1)) / 3) bytes in it. Returns the number of bytes written -// to [out]. -int base64Encode(const unsigned char *in, int inLen, char *out); - -// Compute HMAC-SHA-1 with key [key] and message [message], storing result -// in [hmac] -void HMAC_SHA1(unsigned char hmac[20], const unsigned char *key, int key_len, - const unsigned char *message, int message_len); - -// Compute a 64-bit hash values given a set of bytes -uint64_t hash(const unsigned char *k, int length); - // Because Windows seems to be missing isblank(), use our own; it's a very // easy function to write in any case int is_blank(char c); diff --git a/src/acl.c b/src/acl.c deleted file mode 100644 index b27b8db..0000000 --- a/src/acl.c +++ /dev/null @@ -1,350 +0,0 @@ -/** ************************************************************************** - * acl.c - * - * Copyright 2008 Bryan Ischo - * - * This file is part of libs3. - * - * libs3 is free software: you can redistribute it and/or modify it under the - * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. - * - * In addition, as a special exception, the copyright holders give - * permission to link the code of this library and its programs with the - * OpenSSL library, and distribute linked combinations including the two. - * - * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY - * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU Lesser General Public License - * version 3 along with libs3, in a file named COPYING. If not, see - * . - * - ************************************************************************** **/ - -#include -#include -#include "libs3.h" -#include "request.h" - -// Use a rather arbitrary max size for the document of 64K -#define ACL_XML_DOC_MAXSIZE (64 * 1024) - - -// get acl ------------------------------------------------------------------- - -typedef struct GetAclData -{ - SimpleXml simpleXml; - - S3ResponsePropertiesCallback *responsePropertiesCallback; - S3ResponseCompleteCallback *responseCompleteCallback; - void *callbackData; - - int *aclGrantCountReturn; - S3AclGrant *aclGrants; - char *ownerId; - char *ownerDisplayName; - string_buffer(aclXmlDocument, ACL_XML_DOC_MAXSIZE); -} GetAclData; - - -static S3Status getAclPropertiesCallback - (const S3ResponseProperties *responseProperties, void *callbackData) -{ - GetAclData *gaData = (GetAclData *) callbackData; - - return (*(gaData->responsePropertiesCallback)) - (responseProperties, gaData->callbackData); -} - - -static S3Status getAclDataCallback(int bufferSize, const char *buffer, - void *callbackData) -{ - GetAclData *gaData = (GetAclData *) callbackData; - - int fit; - - string_buffer_append(gaData->aclXmlDocument, buffer, bufferSize, fit); - - return fit ? S3StatusOK : S3StatusXmlDocumentTooLarge; -} - - -static void getAclCompleteCallback(S3Status requestStatus, - const S3ErrorDetails *s3ErrorDetails, - void *callbackData) -{ - GetAclData *gaData = (GetAclData *) callbackData; - - if (requestStatus == S3StatusOK) { - // Parse the document - requestStatus = S3_convert_acl - (gaData->aclXmlDocument, gaData->ownerId, gaData->ownerDisplayName, - gaData->aclGrantCountReturn, gaData->aclGrants); - } - - (*(gaData->responseCompleteCallback)) - (requestStatus, s3ErrorDetails, gaData->callbackData); - - free(gaData); -} - - -void S3_get_acl(const S3BucketContext *bucketContext, const char *key, - char *ownerId, char *ownerDisplayName, - int *aclGrantCountReturn, S3AclGrant *aclGrants, - S3RequestContext *requestContext, - const S3ResponseHandler *handler, void *callbackData) -{ - // Create the callback data - GetAclData *gaData = (GetAclData *) malloc(sizeof(GetAclData)); - if (!gaData) { - (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData); - return; - } - - gaData->responsePropertiesCallback = handler->propertiesCallback; - gaData->responseCompleteCallback = handler->completeCallback; - gaData->callbackData = callbackData; - - gaData->aclGrantCountReturn = aclGrantCountReturn; - gaData->aclGrants = aclGrants; - gaData->ownerId = ownerId; - gaData->ownerDisplayName = ownerDisplayName; - string_buffer_initialize(gaData->aclXmlDocument); - *aclGrantCountReturn = 0; - - // Set up the RequestParams - RequestParams params = - { - HttpRequestTypeGET, // httpRequestType - { bucketContext->hostName, // hostName - bucketContext->bucketName, // bucketName - bucketContext->protocol, // protocol - bucketContext->uriStyle, // uriStyle - bucketContext->accessKeyId, // accessKeyId - bucketContext->secretAccessKey, // secretAccessKey - bucketContext->securityToken }, // securityToken - key, // key - 0, // queryParams - "acl", // subResource - 0, // copySourceBucketName - 0, // copySourceKey - 0, // getConditions - 0, // startByte - 0, // byteCount - 0, // putProperties - &getAclPropertiesCallback, // propertiesCallback - 0, // toS3Callback - 0, // toS3CallbackTotalSize - &getAclDataCallback, // fromS3Callback - &getAclCompleteCallback, // completeCallback - gaData // callbackData - }; - - // Perform the request - request_perform(¶ms, requestContext); -} - - -// set acl ------------------------------------------------------------------- - -static S3Status generateAclXmlDocument(const char *ownerId, - const char *ownerDisplayName, - int aclGrantCount, - const S3AclGrant *aclGrants, - int *xmlDocumentLenReturn, - char *xmlDocument, - int xmlDocumentBufferSize) -{ - *xmlDocumentLenReturn = 0; - -#define append(fmt, ...) \ - do { \ - *xmlDocumentLenReturn += snprintf \ - (&(xmlDocument[*xmlDocumentLenReturn]), \ - xmlDocumentBufferSize - *xmlDocumentLenReturn - 1, \ - fmt, __VA_ARGS__); \ - if (*xmlDocumentLenReturn >= xmlDocumentBufferSize) { \ - return S3StatusXmlDocumentTooLarge; \ - } \ - } while (0) - - append("%s%s" - "", ownerId, - ownerDisplayName); - - int i; - for (i = 0; i < aclGrantCount; i++) { - append("%s", "granteeType) { - case S3GranteeTypeAmazonCustomerByEmail: - append("AmazonCustomerByEmail\">%s", - grant->grantee.amazonCustomerByEmail.emailAddress); - break; - case S3GranteeTypeCanonicalUser: - append("CanonicalUser\">%s%s", - grant->grantee.canonicalUser.id, - grant->grantee.canonicalUser.displayName); - break; - default: { // case S3GranteeTypeAllAwsUsers/S3GranteeTypeAllUsers: - const char *grantee; - switch (grant->granteeType) { - case S3GranteeTypeAllAwsUsers: - grantee = ACS_GROUP_AWS_USERS; - break; - case S3GranteeTypeAllUsers: - grantee = ACS_GROUP_ALL_USERS; - break; - default: - grantee = ACS_GROUP_LOG_DELIVERY; - break; - } - append("Group\">%s", grantee); - } - break; - } - append("%s", - ((grant->permission == S3PermissionRead) ? "READ" : - (grant->permission == S3PermissionWrite) ? "WRITE" : - (grant->permission == S3PermissionReadACP) ? "READ_ACP" : - (grant->permission == S3PermissionWriteACP) ? "WRITE_ACP" : - "FULL_CONTROL")); - } - - append("%s", ""); - - return S3StatusOK; -} - - -typedef struct SetAclData -{ - S3ResponsePropertiesCallback *responsePropertiesCallback; - S3ResponseCompleteCallback *responseCompleteCallback; - void *callbackData; - - int aclXmlDocumentLen; - char aclXmlDocument[ACL_XML_DOC_MAXSIZE]; - int aclXmlDocumentBytesWritten; - -} SetAclData; - - -static S3Status setAclPropertiesCallback - (const S3ResponseProperties *responseProperties, void *callbackData) -{ - SetAclData *paData = (SetAclData *) callbackData; - - return (*(paData->responsePropertiesCallback)) - (responseProperties, paData->callbackData); -} - - -static int setAclDataCallback(int bufferSize, char *buffer, void *callbackData) -{ - SetAclData *paData = (SetAclData *) callbackData; - - int remaining = (paData->aclXmlDocumentLen - - paData->aclXmlDocumentBytesWritten); - - int toCopy = bufferSize > remaining ? remaining : bufferSize; - - if (!toCopy) { - return 0; - } - - memcpy(buffer, &(paData->aclXmlDocument - [paData->aclXmlDocumentBytesWritten]), toCopy); - - paData->aclXmlDocumentBytesWritten += toCopy; - - return toCopy; -} - - -static void setAclCompleteCallback(S3Status requestStatus, - const S3ErrorDetails *s3ErrorDetails, - void *callbackData) -{ - SetAclData *paData = (SetAclData *) callbackData; - - (*(paData->responseCompleteCallback)) - (requestStatus, s3ErrorDetails, paData->callbackData); - - free(paData); -} - - -void S3_set_acl(const S3BucketContext *bucketContext, const char *key, - const char *ownerId, const char *ownerDisplayName, - int aclGrantCount, const S3AclGrant *aclGrants, - S3RequestContext *requestContext, - const S3ResponseHandler *handler, void *callbackData) -{ - if (aclGrantCount > S3_MAX_ACL_GRANT_COUNT) { - (*(handler->completeCallback)) - (S3StatusTooManyGrants, 0, callbackData); - return; - } - - SetAclData *data = (SetAclData *) malloc(sizeof(SetAclData)); - if (!data) { - (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData); - return; - } - - // Convert aclGrants to XML document - S3Status status = generateAclXmlDocument - (ownerId, ownerDisplayName, aclGrantCount, aclGrants, - &(data->aclXmlDocumentLen), data->aclXmlDocument, - sizeof(data->aclXmlDocument)); - if (status != S3StatusOK) { - free(data); - (*(handler->completeCallback))(status, 0, callbackData); - return; - } - - data->responsePropertiesCallback = handler->propertiesCallback; - data->responseCompleteCallback = handler->completeCallback; - data->callbackData = callbackData; - - data->aclXmlDocumentBytesWritten = 0; - - // Set up the RequestParams - RequestParams params = - { - HttpRequestTypePUT, // httpRequestType - { bucketContext->hostName, // hostName - bucketContext->bucketName, // bucketName - bucketContext->protocol, // protocol - bucketContext->uriStyle, // uriStyle - bucketContext->accessKeyId, // accessKeyId - bucketContext->secretAccessKey, // secretAccessKey - bucketContext->securityToken }, // securityToken - key, // key - 0, // queryParams - "acl", // subResource - 0, // copySourceBucketName - 0, // copySourceKey - 0, // getConditions - 0, // startByte - 0, // byteCount - 0, // putProperties - &setAclPropertiesCallback, // propertiesCallback - &setAclDataCallback, // toS3Callback - data->aclXmlDocumentLen, // toS3CallbackTotalSize - 0, // fromS3Callback - &setAclCompleteCallback, // completeCallback - data // callbackData - }; - - // Perform the request - request_perform(¶ms, requestContext); -} diff --git a/src/bucket.c b/src/bucket.c index dcbcf2f..6d30674 100644 --- a/src/bucket.c +++ b/src/bucket.c @@ -1,13 +1,15 @@ /** ************************************************************************** * bucket.c - * + * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include @@ -70,7 +76,7 @@ static S3Status testBucketPropertiesCallback (const S3ResponseProperties *responseProperties, void *callbackData) { TestBucketData *tbData = (TestBucketData *) callbackData; - + return (*(tbData->responsePropertiesCallback)) (responseProperties, tbData->callbackData); } @@ -85,15 +91,15 @@ static S3Status testBucketDataCallback(int bufferSize, const char *buffer, } -static void testBucketCompleteCallback(S3Status requestStatus, +static void testBucketCompleteCallback(S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, void *callbackData) { TestBucketData *tbData = (TestBucketData *) callbackData; // Copy the location constraint into the return buffer - snprintf(tbData->locationConstraintReturn, - tbData->locationConstraintReturnSize, "%s", + snprintf(tbData->locationConstraintReturn, + tbData->locationConstraintReturnSize, "%s", tbData->locationConstraint); (*(tbData->responseCompleteCallback)) @@ -107,13 +113,15 @@ static void testBucketCompleteCallback(S3Status requestStatus, void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle, const char *accessKeyId, const char *secretAccessKey, const char *securityToken, const char *hostName, - const char *bucketName, int locationConstraintReturnSize, + const char *bucketName, const char *authRegion, + int locationConstraintReturnSize, char *locationConstraintReturn, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData) { // Create the callback data - TestBucketData *tbData = + TestBucketData *tbData = (TestBucketData *) malloc(sizeof(TestBucketData)); if (!tbData) { (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData); @@ -140,7 +148,8 @@ void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle, uriStyle, // uriStyle accessKeyId, // accessKeyId secretAccessKey, // secretAccessKey - securityToken }, // securityToken + securityToken, // securityToken + authRegion }, // authRegion 0, // key 0, // queryParams "location", // subResource @@ -155,7 +164,8 @@ void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle, 0, // toS3CallbackTotalSize &testBucketDataCallback, // fromS3Callback &testBucketCompleteCallback, // completeCallback - tbData // callbackData + tbData, // callbackData + timeoutMs // timeoutMs }; // Perform the request @@ -173,20 +183,20 @@ typedef struct CreateBucketData char doc[1024]; int docLen, docBytesWritten; -} CreateBucketData; - +} CreateBucketData; + static S3Status createBucketPropertiesCallback (const S3ResponseProperties *responseProperties, void *callbackData) { CreateBucketData *cbData = (CreateBucketData *) callbackData; - + return (*(cbData->responsePropertiesCallback)) (responseProperties, cbData->callbackData); } -static int createBucketDataCallback(int bufferSize, char *buffer, +static int createBucketDataCallback(int bufferSize, char *buffer, void *callbackData) { CreateBucketData *cbData = (CreateBucketData *) callbackData; @@ -198,7 +208,7 @@ static int createBucketDataCallback(int bufferSize, char *buffer, int remaining = (cbData->docLen - cbData->docBytesWritten); int toCopy = bufferSize > remaining ? remaining : bufferSize; - + if (!toCopy) { return 0; } @@ -211,7 +221,7 @@ static int createBucketDataCallback(int bufferSize, char *buffer, } -static void createBucketCompleteCallback(S3Status requestStatus, +static void createBucketCompleteCallback(S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, void *callbackData) { @@ -223,16 +233,29 @@ static void createBucketCompleteCallback(S3Status requestStatus, free(cbData); } +static S3Status createBucketFromS3Callback(int bufferSize, const char *buffer, + void *callbackData) +{ + // Sometimes S3 sends response body. We sillently ignore it. + + (void)bufferSize; // avoid unused parameter warning + (void)buffer; // avoid unused parameter warning + (void)callbackData; // avoid unused parameter warning + + return S3StatusOK; +} void S3_create_bucket(S3Protocol protocol, const char *accessKeyId, const char *secretAccessKey, const char *securityToken, const char *hostName, const char *bucketName, - S3CannedAcl cannedAcl, const char *locationConstraint, + const char *authRegion, S3CannedAcl cannedAcl, + const char *locationConstraint, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData) { // Create the callback data - CreateBucketData *cbData = + CreateBucketData *cbData = (CreateBucketData *) malloc(sizeof(CreateBucketData)); if (!cbData) { (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData); @@ -254,7 +277,7 @@ void S3_create_bucket(S3Protocol protocol, const char *accessKeyId, else { cbData->docLen = 0; } - + // Set up S3PutProperties S3PutProperties properties = { @@ -269,7 +292,7 @@ void S3_create_bucket(S3Protocol protocol, const char *accessKeyId, 0, // metaData 0 // useServerSideEncryption }; - + // Set up the RequestParams RequestParams params = { @@ -280,7 +303,8 @@ void S3_create_bucket(S3Protocol protocol, const char *accessKeyId, S3UriStylePath, // uriStyle accessKeyId, // accessKeyId secretAccessKey, // secretAccessKey - securityToken }, // securityToken + securityToken, // securityToken + authRegion }, // authRegion 0, // key 0, // queryParams 0, // subResource @@ -293,16 +317,17 @@ void S3_create_bucket(S3Protocol protocol, const char *accessKeyId, &createBucketPropertiesCallback, // propertiesCallback &createBucketDataCallback, // toS3Callback cbData->docLen, // toS3CallbackTotalSize - 0, // fromS3Callback + createBucketFromS3Callback, // fromS3Callback &createBucketCompleteCallback, // completeCallback - cbData // callbackData + cbData, // callbackData + timeoutMs // timeoutMs }; // Perform the request request_perform(¶ms, requestContext); } - + // delete bucket ------------------------------------------------------------- typedef struct DeleteBucketData @@ -317,13 +342,13 @@ static S3Status deleteBucketPropertiesCallback (const S3ResponseProperties *responseProperties, void *callbackData) { DeleteBucketData *dbData = (DeleteBucketData *) callbackData; - + return (*(dbData->responsePropertiesCallback)) (responseProperties, dbData->callbackData); } -static void deleteBucketCompleteCallback(S3Status requestStatus, +static void deleteBucketCompleteCallback(S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, void *callbackData) { @@ -338,12 +363,14 @@ static void deleteBucketCompleteCallback(S3Status requestStatus, void S3_delete_bucket(S3Protocol protocol, S3UriStyle uriStyle, const char *accessKeyId, const char *secretAccessKey, - const char *securityToken, const char *hostName, - const char *bucketName, S3RequestContext *requestContext, + const char *securityToken, const char *hostName, + const char *bucketName, const char *authRegion, + S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData) { // Create the callback data - DeleteBucketData *dbData = + DeleteBucketData *dbData = (DeleteBucketData *) malloc(sizeof(DeleteBucketData)); if (!dbData) { (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData); @@ -364,7 +391,8 @@ void S3_delete_bucket(S3Protocol protocol, S3UriStyle uriStyle, uriStyle, // uriStyle accessKeyId, // accessKeyId secretAccessKey, // secretAccessKey - securityToken }, // securityToken + securityToken, // securityToken + authRegion }, // authRegion 0, // key 0, // queryParams 0, // subResource @@ -379,7 +407,8 @@ void S3_delete_bucket(S3Protocol protocol, S3UriStyle uriStyle, 0, // toS3CallbackTotalSize 0, // fromS3Callback &deleteBucketCompleteCallback, // completeCallback - dbData // callbackData + dbData, // callbackData + timeoutMs // timeoutMs }; // Perform the request @@ -462,7 +491,7 @@ static S3Status make_list_bucket_callback(ListBucketData *lbData) S3ListBucketContent *contentDest = &(contents[i]); ListBucketContents *contentSrc = &(lbData->contents[i]); contentDest->key = contentSrc->key; - contentDest->lastModified = + contentDest->lastModified = parseIso8601Time(contentSrc->lastModified); contentDest->eTag = contentSrc->eTag; contentDest->size = parseUnsignedInt(contentSrc->size); @@ -481,7 +510,7 @@ static S3Status make_list_bucket_callback(ListBucketData *lbData) return (*(lbData->listBucketCallback)) (isTruncated, lbData->nextMarker, - contentsCount, contents, commonPrefixesCount, + contentsCount, contents, commonPrefixesCount, (const char **) commonPrefixes, lbData->callbackData); } @@ -502,45 +531,46 @@ static S3Status listBucketXmlCallback(const char *elementPath, string_buffer_append(lbData->nextMarker, data, dataLen, fit); } else if (!strcmp(elementPath, "ListBucketResult/Contents/Key")) { - ListBucketContents *contents = + ListBucketContents *contents = &(lbData->contents[lbData->contentsCount]); string_buffer_append(contents->key, data, dataLen, fit); } - else if (!strcmp(elementPath, + else if (!strcmp(elementPath, "ListBucketResult/Contents/LastModified")) { - ListBucketContents *contents = + ListBucketContents *contents = &(lbData->contents[lbData->contentsCount]); string_buffer_append(contents->lastModified, data, dataLen, fit); } else if (!strcmp(elementPath, "ListBucketResult/Contents/ETag")) { - ListBucketContents *contents = + ListBucketContents *contents = &(lbData->contents[lbData->contentsCount]); string_buffer_append(contents->eTag, data, dataLen, fit); } else if (!strcmp(elementPath, "ListBucketResult/Contents/Size")) { - ListBucketContents *contents = + ListBucketContents *contents = &(lbData->contents[lbData->contentsCount]); string_buffer_append(contents->size, data, dataLen, fit); } else if (!strcmp(elementPath, "ListBucketResult/Contents/Owner/ID")) { - ListBucketContents *contents = + ListBucketContents *contents = &(lbData->contents[lbData->contentsCount]); string_buffer_append(contents->ownerId, data, dataLen, fit); } - else if (!strcmp(elementPath, + else if (!strcmp(elementPath, "ListBucketResult/Contents/Owner/DisplayName")) { - ListBucketContents *contents = + ListBucketContents *contents = &(lbData->contents[lbData->contentsCount]); string_buffer_append (contents->ownerDisplayName, data, dataLen, fit); } - else if (!strcmp(elementPath, + else if (!strcmp(elementPath, "ListBucketResult/CommonPrefixes/Prefix")) { int which = lbData->commonPrefixesCount; + size_t oldLen = lbData->commonPrefixLens[which]; lbData->commonPrefixLens[which] += - snprintf(lbData->commonPrefixes[which], + snprintf(lbData->commonPrefixes[which]+oldLen, sizeof(lbData->commonPrefixes[which]) - - lbData->commonPrefixLens[which] - 1, + oldLen - 1, "%.*s", dataLen, data); if (lbData->commonPrefixLens[which] >= (int) sizeof(lbData->commonPrefixes[which])) { @@ -597,22 +627,22 @@ static S3Status listBucketPropertiesCallback (const S3ResponseProperties *responseProperties, void *callbackData) { ListBucketData *lbData = (ListBucketData *) callbackData; - + return (*(lbData->responsePropertiesCallback)) (responseProperties, lbData->callbackData); } -static S3Status listBucketDataCallback(int bufferSize, const char *buffer, +static S3Status listBucketDataCallback(int bufferSize, const char *buffer, void *callbackData) { ListBucketData *lbData = (ListBucketData *) callbackData; - + return simplexml_add(&(lbData->simpleXml), buffer, bufferSize); } -static void listBucketCompleteCallback(S3Status requestStatus, +static void listBucketCompleteCallback(S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, void *callbackData) { @@ -635,12 +665,13 @@ static void listBucketCompleteCallback(S3Status requestStatus, void S3_list_bucket(const S3BucketContext *bucketContext, const char *prefix, const char *marker, const char *delimiter, int maxkeys, S3RequestContext *requestContext, + int timeoutMs, const S3ListBucketHandler *handler, void *callbackData) { // Compose the query params string_buffer(queryParams, 4096); string_buffer_initialize(queryParams); - + #define safe_append(name, value) \ do { \ int fit; \ @@ -661,7 +692,7 @@ void S3_list_bucket(const S3BucketContext *bucketContext, const char *prefix, } \ amp = 1; \ char encoded[3 * 1024]; \ - if (!urlEncode(encoded, value, 1024)) { \ + if (!urlEncode(encoded, value, 1024, 1)) { \ (*(handler->responseHandler.completeCallback)) \ (S3StatusQueryParamsTooLong, 0, callbackData); \ return; \ @@ -702,11 +733,11 @@ void S3_list_bucket(const S3BucketContext *bucketContext, const char *prefix, } simplexml_initialize(&(lbData->simpleXml), &listBucketXmlCallback, lbData); - - lbData->responsePropertiesCallback = + + lbData->responsePropertiesCallback = handler->responseHandler.propertiesCallback; lbData->listBucketCallback = handler->listBucketCallback; - lbData->responseCompleteCallback = + lbData->responseCompleteCallback = handler->responseHandler.completeCallback; lbData->callbackData = callbackData; @@ -724,7 +755,8 @@ void S3_list_bucket(const S3BucketContext *bucketContext, const char *prefix, bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId bucketContext->secretAccessKey, // secretAccessKey - bucketContext->securityToken }, // securityToken + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion 0, // key queryParams[0] ? queryParams : 0, // queryParams 0, // subResource @@ -739,7 +771,8 @@ void S3_list_bucket(const S3BucketContext *bucketContext, const char *prefix, 0, // toS3CallbackTotalSize &listBucketDataCallback, // fromS3Callback &listBucketCompleteCallback, // completeCallback - lbData // callbackData + lbData, // callbackData + timeoutMs // timeoutMs }; // Perform the request diff --git a/src/bucket_metadata.c b/src/bucket_metadata.c new file mode 100644 index 0000000..05f3e67 --- /dev/null +++ b/src/bucket_metadata.c @@ -0,0 +1,608 @@ +/** ************************************************************************** + * bucket_metadata.c + * + * Copyright 2008 Bryan Ischo + * + * This file is part of libs3. + * + * libs3 is free software: you can redistribute it and/or modify it under the + * terms of the GNU Lesser General Public License as published by the Free + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. + * + * In addition, as a special exception, the copyright holders give + * permission to link the code of this library and its programs with the + * OpenSSL library, and distribute linked combinations including the two. + * + * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 3 along with libs3, in a file named COPYING. If not, see + * . + * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * + ************************************************************************** **/ + +#include +#include + +#ifndef __APPLE__ + #include + #include + #include + #include +#endif + +#include "libs3.h" +#include "request.h" + +// Use a rather arbitrary max size for the document of 64K +#define ACL_XML_DOC_MAXSIZE (64 * 1024) + + +// get acl ------------------------------------------------------------------- + +typedef struct GetAclData +{ + SimpleXml simpleXml; + + S3ResponsePropertiesCallback *responsePropertiesCallback; + S3ResponseCompleteCallback *responseCompleteCallback; + void *callbackData; + + int *aclGrantCountReturn; + S3AclGrant *aclGrants; + char *ownerId; + char *ownerDisplayName; + string_buffer(xmlDocument, ACL_XML_DOC_MAXSIZE); +} GetAclData; + + +static S3Status getAclPropertiesCallback + (const S3ResponseProperties *responseProperties, void *callbackData) +{ + GetAclData *gaData = (GetAclData *) callbackData; + + return (*(gaData->responsePropertiesCallback)) + (responseProperties, gaData->callbackData); +} + + +static S3Status getAclDataCallback(int bufferSize, const char *buffer, + void *callbackData) +{ + GetAclData *gaData = (GetAclData *) callbackData; + + int fit; + + string_buffer_append(gaData->xmlDocument, buffer, bufferSize, fit); + + return fit ? S3StatusOK : S3StatusXmlDocumentTooLarge; +} + + +static void getAclCompleteCallback(S3Status requestStatus, + const S3ErrorDetails *s3ErrorDetails, + void *callbackData) +{ + GetAclData *gaData = (GetAclData *) callbackData; + + if (requestStatus == S3StatusOK) { + // Parse the document + requestStatus = S3_convert_acl + (gaData->xmlDocument, gaData->ownerId, gaData->ownerDisplayName, + gaData->aclGrantCountReturn, gaData->aclGrants); + } + + (*(gaData->responseCompleteCallback)) + (requestStatus, s3ErrorDetails, gaData->callbackData); + + free(gaData); +} + + +void S3_get_acl(const S3BucketContext *bucketContext, const char *key, + char *ownerId, char *ownerDisplayName, + int *aclGrantCountReturn, S3AclGrant *aclGrants, + S3RequestContext *requestContext, + int timeoutMs, + const S3ResponseHandler *handler, void *callbackData) +{ + // Create the callback data + GetAclData *gaData = (GetAclData *) malloc(sizeof(GetAclData)); + if (!gaData) { + (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData); + return; + } + + gaData->responsePropertiesCallback = handler->propertiesCallback; + gaData->responseCompleteCallback = handler->completeCallback; + gaData->callbackData = callbackData; + + gaData->aclGrantCountReturn = aclGrantCountReturn; + gaData->aclGrants = aclGrants; + gaData->ownerId = ownerId; + gaData->ownerDisplayName = ownerDisplayName; + string_buffer_initialize(gaData->xmlDocument); + *aclGrantCountReturn = 0; + + // Set up the RequestParams + RequestParams params = + { + HttpRequestTypeGET, // httpRequestType + { bucketContext->hostName, // hostName + bucketContext->bucketName, // bucketName + bucketContext->protocol, // protocol + bucketContext->uriStyle, // uriStyle + bucketContext->accessKeyId, // accessKeyId + bucketContext->secretAccessKey, // secretAccessKey + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion + key, // key + 0, // queryParams + "acl", // subResource + 0, // copySourceBucketName + 0, // copySourceKey + 0, // getConditions + 0, // startByte + 0, // byteCount + 0, // putProperties + &getAclPropertiesCallback, // propertiesCallback + 0, // toS3Callback + 0, // toS3CallbackTotalSize + &getAclDataCallback, // fromS3Callback + &getAclCompleteCallback, // completeCallback + gaData, // callbackData + timeoutMs // timeoutMs + }; + + // Perform the request + request_perform(¶ms, requestContext); +} + + +// set acl ------------------------------------------------------------------- + +static S3Status generateAclXmlDocument(const char *ownerId, + const char *ownerDisplayName, + int aclGrantCount, + const S3AclGrant *aclGrants, + int *xmlDocumentLenReturn, + char *xmlDocument, + int xmlDocumentBufferSize) +{ + *xmlDocumentLenReturn = 0; + +#define append(fmt, ...) \ + do { \ + *xmlDocumentLenReturn += snprintf \ + (&(xmlDocument[*xmlDocumentLenReturn]), \ + xmlDocumentBufferSize - *xmlDocumentLenReturn - 1, \ + fmt, __VA_ARGS__); \ + if (*xmlDocumentLenReturn >= xmlDocumentBufferSize) { \ + return S3StatusXmlDocumentTooLarge; \ + } \ + } while (0) + + append("%s%s" + "", ownerId, + ownerDisplayName); + + int i; + for (i = 0; i < aclGrantCount; i++) { + append("%s", "granteeType) { + case S3GranteeTypeAmazonCustomerByEmail: + append("AmazonCustomerByEmail\">%s", + grant->grantee.amazonCustomerByEmail.emailAddress); + break; + case S3GranteeTypeCanonicalUser: + append("CanonicalUser\">%s%s", + grant->grantee.canonicalUser.id, + grant->grantee.canonicalUser.displayName); + break; + default: { // case S3GranteeTypeAllAwsUsers/S3GranteeTypeAllUsers: + const char *grantee; + switch (grant->granteeType) { + case S3GranteeTypeAllAwsUsers: + grantee = ACS_GROUP_AWS_USERS; + break; + case S3GranteeTypeAllUsers: + grantee = ACS_GROUP_ALL_USERS; + break; + default: + grantee = ACS_GROUP_LOG_DELIVERY; + break; + } + append("Group\">%s", grantee); + } + break; + } + append("%s", + ((grant->permission == S3PermissionRead) ? "READ" : + (grant->permission == S3PermissionWrite) ? "WRITE" : + (grant->permission == S3PermissionReadACP) ? "READ_ACP" : + (grant->permission == S3PermissionWriteACP) ? "WRITE_ACP" : + "FULL_CONTROL")); + } + + append("%s", ""); + + return S3StatusOK; +} + + +typedef struct SetXmlData +{ + S3ResponsePropertiesCallback *responsePropertiesCallback; + S3ResponseCompleteCallback *responseCompleteCallback; + void *callbackData; + + int xmlDocumentLen; + const char *xmlDocument; + int xmlDocumentBytesWritten; + +} SetXmlData; + + +static S3Status setXmlPropertiesCallback + (const S3ResponseProperties *responseProperties, void *callbackData) +{ + SetXmlData *paData = (SetXmlData *) callbackData; + + return (*(paData->responsePropertiesCallback)) + (responseProperties, paData->callbackData); +} + + +static int setXmlDataCallback(int bufferSize, char *buffer, void *callbackData) +{ + SetXmlData *paData = (SetXmlData *) callbackData; + + int remaining = (paData->xmlDocumentLen - + paData->xmlDocumentBytesWritten); + + int toCopy = bufferSize > remaining ? remaining : bufferSize; + + if (!toCopy) { + return 0; + } + + memcpy(buffer, &(paData->xmlDocument + [paData->xmlDocumentBytesWritten]), toCopy); + + paData->xmlDocumentBytesWritten += toCopy; + + return toCopy; +} + + +static void setXmlCompleteCallback(S3Status requestStatus, + const S3ErrorDetails *s3ErrorDetails, + void *callbackData) +{ + SetXmlData *paData = (SetXmlData *) callbackData; + + (*(paData->responseCompleteCallback)) + (requestStatus, s3ErrorDetails, paData->callbackData); + + free(paData); +} + + +void S3_set_acl(const S3BucketContext *bucketContext, const char *key, + const char *ownerId, const char *ownerDisplayName, + int aclGrantCount, const S3AclGrant *aclGrants, + S3RequestContext *requestContext, + int timeoutMs, + const S3ResponseHandler *handler, void *callbackData) +{ + char aclBuffer[ACL_XML_DOC_MAXSIZE]; + + if (aclGrantCount > S3_MAX_ACL_GRANT_COUNT) { + (*(handler->completeCallback)) + (S3StatusTooManyGrants, 0, callbackData); + return; + } + + SetXmlData *data = (SetXmlData *) malloc(sizeof(SetXmlData)); + if (!data) { + (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData); + return; + } + + data->xmlDocument = aclBuffer; + + // Convert aclGrants to XML document + S3Status status = generateAclXmlDocument + (ownerId, ownerDisplayName, aclGrantCount, aclGrants, + &(data->xmlDocumentLen), aclBuffer, + sizeof(aclBuffer)); + if (status != S3StatusOK) { + free(data); + (*(handler->completeCallback))(status, 0, callbackData); + return; + } + + data->responsePropertiesCallback = handler->propertiesCallback; + data->responseCompleteCallback = handler->completeCallback; + data->callbackData = callbackData; + + data->xmlDocumentBytesWritten = 0; + + // Set up the RequestParams + RequestParams params = + { + HttpRequestTypePUT, // httpRequestType + { bucketContext->hostName, // hostName + bucketContext->bucketName, // bucketName + bucketContext->protocol, // protocol + bucketContext->uriStyle, // uriStyle + bucketContext->accessKeyId, // accessKeyId + bucketContext->secretAccessKey, // secretAccessKey + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion + key, // key + 0, // queryParams + "acl", // subResource + 0, // copySourceBucketName + 0, // copySourceKey + 0, // getConditions + 0, // startByte + 0, // byteCount + 0, // putProperties + &setXmlPropertiesCallback, // propertiesCallback + &setXmlDataCallback, // toS3Callback + data->xmlDocumentLen, // toS3CallbackTotalSize + 0, // fromS3Callback + &setXmlCompleteCallback, // completeCallback + data, // callbackData + timeoutMs // timeoutMs + }; + + // Perform the request + request_perform(¶ms, requestContext); +} + + +// get lifecycle ------------------------------------------------------------------- + +typedef struct GetLifecycleData +{ + S3ResponsePropertiesCallback *responsePropertiesCallback; + S3ResponseCompleteCallback *responseCompleteCallback; + void *callbackData; + + char *lifecycleXmlDocumentReturn; + int lifecycleXmlDocumentBufferSize; + int lifecycleXmlDocumentWritten; +} GetLifecycleData; + + +static S3Status getLifecyclePropertiesCallback + (const S3ResponseProperties *responseProperties, void *callbackData) +{ + GetLifecycleData *gaData = (GetLifecycleData *) callbackData; + + return (*(gaData->responsePropertiesCallback)) + (responseProperties, gaData->callbackData); +} + + +static S3Status getLifecycleDataCallback(int bufferSize, const char *buffer, + void *callbackData) +{ + GetLifecycleData *gaData = (GetLifecycleData *) callbackData; + + if ((gaData->lifecycleXmlDocumentWritten + bufferSize) >= gaData->lifecycleXmlDocumentBufferSize) + return S3StatusXmlDocumentTooLarge; + + snprintf(gaData->lifecycleXmlDocumentReturn + gaData->lifecycleXmlDocumentWritten, bufferSize + 1, "%s", buffer); + gaData->lifecycleXmlDocumentWritten += bufferSize; + + return S3StatusOK; +} + + +static void getLifecycleCompleteCallback(S3Status requestStatus, + const S3ErrorDetails *s3ErrorDetails, + void *callbackData) +{ + GetLifecycleData *gaData = (GetLifecycleData *) callbackData; + + (*(gaData->responseCompleteCallback)) + (requestStatus, s3ErrorDetails, gaData->callbackData); + + free(gaData); +} + + +void S3_get_lifecycle(const S3BucketContext *bucketContext, + char *lifecycleXmlDocumentReturn, int lifecycleXmlDocumentBufferSize, + S3RequestContext *requestContext, + int timeoutMs, + const S3ResponseHandler *handler, void *callbackData) +{ + // Create the callback data + GetLifecycleData *gaData = (GetLifecycleData *) malloc(sizeof(GetLifecycleData)); + if (!gaData) { + (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData); + return; + } + + gaData->responsePropertiesCallback = handler->propertiesCallback; + gaData->responseCompleteCallback = handler->completeCallback; + gaData->callbackData = callbackData; + + gaData->lifecycleXmlDocumentReturn = lifecycleXmlDocumentReturn; + gaData->lifecycleXmlDocumentBufferSize = lifecycleXmlDocumentBufferSize; + gaData->lifecycleXmlDocumentWritten = 0; + + // Set up the RequestParams + RequestParams params = + { + HttpRequestTypeGET, // httpRequestType + { bucketContext->hostName, // hostName + bucketContext->bucketName, // bucketName + bucketContext->protocol, // protocol + bucketContext->uriStyle, // uriStyle + bucketContext->accessKeyId, // accessKeyId + bucketContext->secretAccessKey, // secretAccessKey + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion + 0, // key + 0, // queryParams + "lifecycle", // subResource + 0, // copySourceBucketName + 0, // copySourceKey + 0, // getConditions + 0, // startByte + 0, // byteCount + 0, // putProperties + &getLifecyclePropertiesCallback, // propertiesCallback + 0, // toS3Callback + 0, // toS3CallbackTotalSize + &getLifecycleDataCallback, // fromS3Callback + &getLifecycleCompleteCallback, // completeCallback + gaData, // callbackData + timeoutMs // timeoutMs + }; + + // Perform the request + request_perform(¶ms, requestContext); +} + + +#ifndef __APPLE__ +// Calculate MD5 and encode it as base64 +void generate_content_md5(const char* data, int size, + char* retBuffer, int retBufferSize) { + MD5_CTX mdContext; + BIO *bio, *b64; + BUF_MEM *bufferPtr; + + char md5Buffer[MD5_DIGEST_LENGTH]; + + MD5_Init(&mdContext); + MD5_Update(&mdContext, data, size); + MD5_Final((unsigned char*)md5Buffer, &mdContext); + + + b64 = BIO_new(BIO_f_base64()); + bio = BIO_new(BIO_s_mem()); + bio = BIO_push(b64, bio); + + BIO_set_flags(bio, BIO_FLAGS_BASE64_NO_NL); //Ignore newlines - write everything in one line + BIO_write(bio, md5Buffer, sizeof(md5Buffer)); + (void) BIO_flush(bio); + BIO_get_mem_ptr(bio, &bufferPtr); + (void) BIO_set_close(bio, BIO_NOCLOSE); + + if ((unsigned int)retBufferSize + 1 < bufferPtr->length) { + retBuffer[0] = '\0'; + BIO_free_all(bio); + return; + } + + memcpy(retBuffer, bufferPtr->data, bufferPtr->length); + retBuffer[bufferPtr->length] = '\0'; + + BIO_free_all(bio); +} +#endif + + +void S3_set_lifecycle(const S3BucketContext *bucketContext, + const char *lifecycleXmlDocument, + S3RequestContext *requestContext, + int timeoutMs, + const S3ResponseHandler *handler, void *callbackData) +{ +#ifdef __APPLE__ + /* This request requires calculating MD5 sum. + * MD5 sum requires OpenSSL library, which is not used on Apple. + * TODO Implement some MD5+Base64 caculation on Apple + */ + (*(handler->completeCallback))(S3StatusNotSupported, 0, callbackData); + return; +#else + char md5Base64[MD5_DIGEST_LENGTH * 2]; + + SetXmlData *data = (SetXmlData *) malloc(sizeof(SetXmlData)); + if (!data) { + (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData); + return; + } + + + data->xmlDocument = lifecycleXmlDocument; + data->xmlDocumentLen = strlen(lifecycleXmlDocument); + + data->responsePropertiesCallback = handler->propertiesCallback; + data->responseCompleteCallback = handler->completeCallback; + data->callbackData = callbackData; + + data->xmlDocumentBytesWritten = 0; + + generate_content_md5(data->xmlDocument, data->xmlDocumentLen, + md5Base64, sizeof (md5Base64)); + + // Set up S3PutProperties + S3PutProperties properties = + { + 0, // contentType + md5Base64, // md5 + 0, // cacheControl + 0, // contentDispositionFilename + 0, // contentEncoding + -1, // expires + 0, // cannedAcl + 0, // metaDataCount + 0, // metaData + 0 // useServerSideEncryption + }; + + // Set up the RequestParams + RequestParams params = + { + HttpRequestTypePUT, // httpRequestType + { bucketContext->hostName, // hostName + bucketContext->bucketName, // bucketName + bucketContext->protocol, // protocol + bucketContext->uriStyle, // uriStyle + bucketContext->accessKeyId, // accessKeyId + bucketContext->secretAccessKey, // secretAccessKey + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion + 0, // key + 0, // queryParams + "lifecycle", // subResource + 0, // copySourceBucketName + 0, // copySourceKey + 0, // getConditions + 0, // startByte + 0, // byteCount + &properties, // putProperties + &setXmlPropertiesCallback, // propertiesCallback + &setXmlDataCallback, // toS3Callback + data->xmlDocumentLen, // toS3CallbackTotalSize + 0, // fromS3Callback + &setXmlCompleteCallback, // completeCallback + data, // callbackData + timeoutMs // timeoutMs + }; + + // Perform the request + request_perform(¶ms, requestContext); +#endif +} + diff --git a/src/error_parser.c b/src/error_parser.c index 7b4aedf..e573aed 100644 --- a/src/error_parser.c +++ b/src/error_parser.c @@ -2,12 +2,14 @@ * error_parser.c * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include @@ -182,6 +188,7 @@ void error_parser_convert_status(ErrorParser *errorParser, S3Status *status) HANDLE_CODE(InvalidBucketName); HANDLE_CODE(InvalidBucketState); HANDLE_CODE(InvalidDigest); + HANDLE_CODE(InvalidEncryptionAlgorithmError); HANDLE_CODE(InvalidLocationConstraint); HANDLE_CODE(InvalidObjectState); HANDLE_CODE(InvalidPart); @@ -217,7 +224,7 @@ void error_parser_convert_status(ErrorParser *errorParser, S3Status *status) HANDLE_CODE(NoSuchVersion); HANDLE_CODE(NotImplemented); HANDLE_CODE(NotSignedUp); - HANDLE_CODE(NotSuchBucketPolicy); + HANDLE_CODE(NoSuchBucketPolicy); HANDLE_CODE(OperationAborted); HANDLE_CODE(PermanentRedirect); HANDLE_CODE(PreconditionFailed); @@ -236,6 +243,7 @@ void error_parser_convert_status(ErrorParser *errorParser, S3Status *status) HANDLE_CODE(UnexpectedContent); HANDLE_CODE(UnresolvableGrantByEmailAddress); HANDLE_CODE(UserKeyMustBeSpecified); + HANDLE_CODE(QuotaExceeded); *status = S3StatusErrorUnknown; code_set: diff --git a/src/general.c b/src/general.c index e5f8696..7876f58 100644 --- a/src/general.c +++ b/src/general.c @@ -1,13 +1,15 @@ /** ************************************************************************** * general.c - * + * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include @@ -107,6 +113,7 @@ const char *S3_get_status_name(S3Status status) handlecase(ServerFailedVerification); handlecase(ConnectionFailed); handlecase(AbortedByCallback); + handlecase(NotSupported); handlecase(ErrorAccessDenied); handlecase(ErrorAccountProblem); handlecase(ErrorAmbiguousGrantByEmailAddress); @@ -119,7 +126,7 @@ const char *S3_get_status_name(S3Status status) handlecase(ErrorEntityTooSmall); handlecase(ErrorEntityTooLarge); handlecase(ErrorExpiredToken); - handlecase(ErrorIllegalVersioningConfigurationException); + handlecase(ErrorIllegalVersioningConfigurationException); handlecase(ErrorIncompleteBody); handlecase(ErrorIncorrectNumberOfFilesInPostRequest); handlecase(ErrorInlineDataTooLarge); @@ -130,8 +137,9 @@ const char *S3_get_status_name(S3Status status) handlecase(ErrorInvalidBucketName); handlecase(ErrorInvalidBucketState); handlecase(ErrorInvalidDigest); + handlecase(ErrorInvalidEncryptionAlgorithmError); handlecase(ErrorInvalidLocationConstraint); - handlecase(ErrorInvalidObjectState); + handlecase(ErrorInvalidObjectState); handlecase(ErrorInvalidPart); handlecase(ErrorInvalidPartOrder); handlecase(ErrorInvalidPayer); @@ -161,11 +169,11 @@ const char *S3_get_status_name(S3Status status) handlecase(ErrorNoSuchBucket); handlecase(ErrorNoSuchKey); handlecase(ErrorNoSuchLifecycleConfiguration); - handlecase(ErrorNoSuchUpload); - handlecase(ErrorNoSuchVersion); + handlecase(ErrorNoSuchUpload); + handlecase(ErrorNoSuchVersion); handlecase(ErrorNotImplemented); handlecase(ErrorNotSignedUp); - handlecase(ErrorNotSuchBucketPolicy); + handlecase(ErrorNoSuchBucketPolicy); handlecase(ErrorOperationAborted); handlecase(ErrorPermanentRedirect); handlecase(ErrorPreconditionFailed); @@ -184,7 +192,8 @@ const char *S3_get_status_name(S3Status status) handlecase(ErrorUnexpectedContent); handlecase(ErrorUnresolvableGrantByEmailAddress); handlecase(ErrorUserKeyMustBeSpecified); - handlecase(ErrorUnknown); + handlecase(ErrorQuotaExceeded); + handlecase(ErrorUnknown); handlecase(HttpErrorMovedTemporarily); handlecase(HttpErrorBadRequest); handlecase(HttpErrorForbidden); @@ -295,7 +304,7 @@ static S3Status convertAclXmlCallback(const char *elementPath, if (data) { if (!strcmp(elementPath, "AccessControlPolicy/Owner/ID")) { - caData->ownerIdLen += + caData->ownerIdLen += snprintf(&(caData->ownerId[caData->ownerIdLen]), S3_MAX_GRANTEE_USER_ID_SIZE - caData->ownerIdLen - 1, "%.*s", dataLen, data); @@ -305,18 +314,18 @@ static S3Status convertAclXmlCallback(const char *elementPath, } else if (!strcmp(elementPath, "AccessControlPolicy/Owner/" "DisplayName")) { - caData->ownerDisplayNameLen += + caData->ownerDisplayNameLen += snprintf(&(caData->ownerDisplayName [caData->ownerDisplayNameLen]), S3_MAX_GRANTEE_DISPLAY_NAME_SIZE - - caData->ownerDisplayNameLen - 1, + caData->ownerDisplayNameLen - 1, "%.*s", dataLen, data); - if (caData->ownerDisplayNameLen >= + if (caData->ownerDisplayNameLen >= S3_MAX_GRANTEE_DISPLAY_NAME_SIZE) { return S3StatusUserDisplayNameTooLong; } } - else if (!strcmp(elementPath, + else if (!strcmp(elementPath, "AccessControlPolicy/AccessControlList/Grant/" "Grantee/EmailAddress")) { // AmazonCustomerByEmail @@ -382,7 +391,7 @@ static S3Status convertAclXmlCallback(const char *elementPath, else if (caData->userId[0] && caData->userDisplayName[0]) { grant->granteeType = S3GranteeTypeCanonicalUser; strcpy(grant->grantee.canonicalUser.id, caData->userId); - strcpy(grant->grantee.canonicalUser.displayName, + strcpy(grant->grantee.canonicalUser.displayName, caData->userDisplayName); } else if (caData->groupUri[0]) { @@ -466,7 +475,7 @@ S3Status S3_convert_acl(char *aclXml, char *ownerId, char *ownerDisplayName, S3Status status = simplexml_add(&simpleXml, aclXml, strlen(aclXml)); simplexml_deinitialize(&simpleXml); - + return status; } diff --git a/src/mingw_functions.c b/src/mingw_functions.c index e8295fe..421f757 100644 --- a/src/mingw_functions.c +++ b/src/mingw_functions.c @@ -2,12 +2,14 @@ * mingw_functions.c * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include diff --git a/src/mingw_s3_functions.c b/src/mingw_s3_functions.c index 675c65a..5098a61 100644 --- a/src/mingw_s3_functions.c +++ b/src/mingw_s3_functions.c @@ -2,12 +2,14 @@ * mingw_s3_functions.c * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ int setenv(const char *a, const char *b, int c) diff --git a/src/multipart.c b/src/multipart.c index b21e0f5..ccff5f8 100644 --- a/src/multipart.c +++ b/src/multipart.c @@ -1,13 +1,15 @@ /** ************************************************************************** * multipart.c - * + * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include @@ -40,8 +46,8 @@ typedef struct InitialMultipartData void *userdata; } InitialMultipartData; -static S3Status InitialMultipartCallback(int bufferSize, const char *buffer, - void *callbackData) +static S3Status InitialMultipartCallback(int bufferSize, const char *buffer, + void *callbackData) { InitialMultipartData *mdata = (InitialMultipartData *) callbackData; return simplexml_add(&(mdata->simpleXml), buffer, bufferSize); @@ -49,7 +55,7 @@ static S3Status InitialMultipartCallback(int bufferSize, const char *buffer, static void InitialMultipartCompleteCallback (S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, - void *callbackData) + void *callbackData) { InitialMultipartData *mdata = (InitialMultipartData *) callbackData; @@ -61,20 +67,20 @@ static void InitialMultipartCompleteCallback if (mdata->handler->responseXmlCallback) { (*mdata->handler->responseXmlCallback) (mdata->upload_id, mdata->userdata); - } + } simplexml_deinitialize(&(mdata->simpleXml)); free(mdata); -} +} static void AbortMultipartUploadCompleteCallback (S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, - void *callbackData) -{ + void *callbackData) +{ (void) callbackData; (void) s3ErrorDetails; fprintf(stderr, "\nERROR: %s\n", S3_get_status_name(requestStatus)); - + } static S3Status initialMultipartXmlCallback(const char *elementPath, @@ -94,14 +100,23 @@ static S3Status initialMultipartXmlCallback(const char *elementPath, return S3StatusOK; } +static S3Status InitialMultipartPropertiesCallback(const S3ResponseProperties *properties, + void *callbackData) +{ + InitialMultipartData *mdata = (InitialMultipartData *)callbackData; + return mdata->handler->responseHandler.propertiesCallback(properties, mdata->userdata); +} + + void S3_initiate_multipart(S3BucketContext *bucketContext, const char *key, - S3PutProperties *putProperties, - S3MultipartInitialHandler *handler, - S3RequestContext *requestContext, - void *callbackData) + S3PutProperties *putProperties, + S3MultipartInitialHandler *handler, + S3RequestContext *requestContext, + int timeoutMs, + void *callbackData) { - InitialMultipartData *mdata = - (InitialMultipartData *) malloc(sizeof(InitialMultipartData)); + InitialMultipartData *mdata = + (InitialMultipartData *) malloc(sizeof(InitialMultipartData)); simplexml_initialize(&(mdata->simpleXml), &initialMultipartXmlCallback, mdata); string_buffer_initialize(mdata->upload_id); @@ -112,12 +127,13 @@ void S3_initiate_multipart(S3BucketContext *bucketContext, const char *key, { HttpRequestTypePOST, // httpRequestType { bucketContext->hostName, // hostName - bucketContext->bucketName, // bucketName + bucketContext->bucketName, // bucketName bucketContext->protocol, // protocol bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId - bucketContext->secretAccessKey, //secretAccessKey - bucketContext->securityToken }, // secretToken + bucketContext->secretAccessKey, // secretAccessKey + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion key, // key 0, // queryParams "uploads", // subResource @@ -127,12 +143,13 @@ void S3_initiate_multipart(S3BucketContext *bucketContext, const char *key, 0, // startByte 0, // byteCount putProperties, // putProperties - handler->responseHandler.propertiesCallback, // propertiesCallback + InitialMultipartPropertiesCallback, // propertiesCallback 0, // toS3Callback 0, // toS3CallbackTotalSize InitialMultipartCallback, // fromS3Callback InitialMultipartCompleteCallback, // completeCallback - mdata // callbackData + mdata, // callbackData + timeoutMs // timeoutMs }; // Perform the request @@ -142,7 +159,8 @@ void S3_initiate_multipart(S3BucketContext *bucketContext, const char *key, void S3_abort_multipart_upload(S3BucketContext *bucketContext, const char *key, const char *uploadId, - S3AbortMultipartUploadHandler *handler) + int timeoutMs, + S3AbortMultipartUploadHandler *handler) { char subResource[512]; snprintf(subResource, 512, "uploadId=%s", uploadId); @@ -155,8 +173,9 @@ void S3_abort_multipart_upload(S3BucketContext *bucketContext, const char *key, bucketContext->protocol, // protocol bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId - bucketContext->secretAccessKey, //secretAccessKey - bucketContext->securityToken }, // secretToken + bucketContext->secretAccessKey, // secretAccessKey + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion key, // key 0, // queryParams subResource, // subResource @@ -171,7 +190,8 @@ void S3_abort_multipart_upload(S3BucketContext *bucketContext, const char *key, 0, // toS3CallbackTotalSize 0, // fromS3Callback AbortMultipartUploadCompleteCallback, // completeCallback - 0 // callbackData + 0, // callbackData + timeoutMs // timeoutMs }; // Perform the request @@ -187,10 +207,12 @@ void S3_upload_part(S3BucketContext *bucketContext, const char *key, S3PutProperties *putProperties, S3PutObjectHandler *handler, int seq, const char *upload_id, int partContentLength, - S3RequestContext *requestContext, void *callbackData) + S3RequestContext *requestContext, + int timeoutMs, + void *callbackData) { - char subResource[512]; - snprintf(subResource, 512, "partNumber=%d&uploadId=%s", seq, upload_id); + char queryParams[512]; + snprintf(queryParams, 512, "partNumber=%d&uploadId=%s", seq, upload_id); RequestParams params = { @@ -200,11 +222,12 @@ void S3_upload_part(S3BucketContext *bucketContext, const char *key, bucketContext->protocol, // protocol bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId - bucketContext->secretAccessKey, //secretAccessKey - bucketContext->securityToken }, // secretToken + bucketContext->secretAccessKey, // secretAccessKey + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion key, // key - 0, // queryParams - subResource, // subResource + queryParams, // queryParams + 0, // subResource 0, // copySourceBucketName 0, // copySourceKey 0, // getConditions @@ -216,7 +239,8 @@ void S3_upload_part(S3BucketContext *bucketContext, const char *key, partContentLength, // toS3CallbackTotalSize 0, // fromS3Callback handler->responseHandler.completeCallback, // completeCallback - callbackData // callbackData + callbackData, // callbackData + timeoutMs // timeoutMs }; request_perform(¶ms, requestContext); @@ -224,7 +248,7 @@ void S3_upload_part(S3BucketContext *bucketContext, const char *key, /* - * S3 commit multipart + * S3 commit multipart * */ @@ -232,15 +256,15 @@ typedef struct CommitMultiPartData { SimpleXml simplexml; void *userdata; S3MultipartCommitHandler *handler; - //response parsed from + //response parsed from string_buffer(location,128); string_buffer(etag,128); } CommitMultiPartData; -static S3Status commitMultipartResponseXMLcallback(const char *elementPath, - const char *data, - int dataLen, +static S3Status commitMultipartResponseXMLcallback(const char *elementPath, + const char *data, + int dataLen, void *callbackData) { int fit; @@ -254,13 +278,13 @@ static S3Status commitMultipartResponseXMLcallback(const char *elementPath, } } (void) fit; - + return S3StatusOK; } -static S3Status commitMultipartCallback(int bufferSize, const char *buffer, - void *callbackData) +static S3Status commitMultipartCallback(int bufferSize, const char *buffer, + void *callbackData) { CommitMultiPartData *data = (CommitMultiPartData *) callbackData; return simplexml_add(&(data->simplexml), buffer, bufferSize); @@ -271,7 +295,7 @@ static S3Status commitMultipartPropertiesCallback (const S3ResponseProperties *responseProperties, void *callbackData) { CommitMultiPartData *data = (CommitMultiPartData *) callbackData; - + if (data->handler->responseHandler.propertiesCallback) { (*(data->handler->responseHandler.propertiesCallback)) (responseProperties, data->userdata); @@ -280,8 +304,8 @@ static S3Status commitMultipartPropertiesCallback } static void commitMultipartCompleteCallback - (S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, - void *callbackData) + (S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, + void *callbackData) { CommitMultiPartData *data = (CommitMultiPartData*) callbackData; if (data->handler->responseHandler.completeCallback) { @@ -297,8 +321,8 @@ static void commitMultipartCompleteCallback } -static int commitMultipartPutObject(int bufferSize, char *buffer, - void *callbackData) +static int commitMultipartPutObject(int bufferSize, char *buffer, + void *callbackData) { CommitMultiPartData *data = (CommitMultiPartData*) callbackData; if (data->handler->putObjectDataCallback) { @@ -312,14 +336,15 @@ static int commitMultipartPutObject(int bufferSize, char *buffer, void S3_complete_multipart_upload(S3BucketContext *bucketContext, const char *key, - S3MultipartCommitHandler *handler, - const char *upload_id, int contentLength, - S3RequestContext *requestContext, + S3MultipartCommitHandler *handler, + const char *upload_id, int contentLength, + S3RequestContext *requestContext, + int timeoutMs, void *callbackData) { - char subResource[512]; - snprintf(subResource, 512, "uploadId=%s", upload_id); - CommitMultiPartData *data = + char queryParams[512]; + snprintf(queryParams, 512, "uploadId=%s", upload_id); + CommitMultiPartData *data = (CommitMultiPartData *) malloc(sizeof(CommitMultiPartData)); data->userdata = callbackData; data->handler = handler; @@ -337,11 +362,12 @@ void S3_complete_multipart_upload(S3BucketContext *bucketContext, bucketContext->protocol, // protocol bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId - bucketContext->secretAccessKey, //secretAccessKey - bucketContext->securityToken }, // secretToken + bucketContext->secretAccessKey, // secretAccessKey + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion key, // key - 0, // queryParams - subResource, // subResource + queryParams, // queryParams + 0, // subResource 0, // copySourceBucketName 0, // copySourceKey 0, // getConditions @@ -353,7 +379,8 @@ void S3_complete_multipart_upload(S3BucketContext *bucketContext, contentLength, // toS3CallbackTotalSize commitMultipartCallback, // fromS3Callback commitMultipartCompleteCallback, // completeCallback - data // callbackData + data, // callbackData + timeoutMs // timeoutMs }; request_perform(¶ms, requestContext); @@ -371,7 +398,7 @@ typedef struct ListMultipartUpload string_buffer(key, 1024); string_buffer(uploadId, 256); string_buffer(initiatorId, 256); - string_buffer(initiatorDisplayName, 256); + string_buffer(initiatorDisplayName, 256); string_buffer(ownerId, 256); string_buffer(ownerDisplayName, 256); string_buffer(storageClass, 256); @@ -384,7 +411,7 @@ typedef struct ListPart string_buffer(eTag, 1024); string_buffer(partNumber, 24); string_buffer(size, 256); - string_buffer(lastModified, 256); + string_buffer(lastModified, 256); } ListPart; @@ -421,12 +448,12 @@ typedef struct ListPartsData string_buffer(isTruncated, 64); string_buffer(nextPartNumberMarker, 1024); - string_buffer(initiatorId, 256); - string_buffer(initiatorDisplayName, 256); + string_buffer(initiatorId, 256); + string_buffer(initiatorDisplayName, 256); string_buffer(ownerId, 256); string_buffer(ownerDisplayName, 256); - string_buffer(storageClass, 256); - + string_buffer(storageClass, 256); + int handlePartsStart; int partsCount; ListPart parts[MAX_PARTS]; @@ -475,7 +502,7 @@ static S3Status listMultipartPropertiesCallback (const S3ResponseProperties *responseProperties, void *callbackData) { ListMultipartData *lmData = (ListMultipartData *) callbackData; - + return (*(lmData->responsePropertiesCallback)) (responseProperties, lmData->callbackData); } @@ -485,26 +512,26 @@ static S3Status listPartsPropertiesCallback (const S3ResponseProperties *responseProperties, void *callbackData) { ListPartsData *lpData = (ListPartsData *) callbackData; - + return (*(lpData->responsePropertiesCallback)) (responseProperties, lpData->callbackData); } -static S3Status listMultipartDataCallback(int bufferSize, const char *buffer, +static S3Status listMultipartDataCallback(int bufferSize, const char *buffer, void *callbackData) { ListMultipartData *lmData = (ListMultipartData *) callbackData; - + return simplexml_add(&(lmData->simpleXml), buffer, bufferSize); } -static S3Status listPartsDataCallback(int bufferSize, const char *buffer, +static S3Status listPartsDataCallback(int bufferSize, const char *buffer, void *callbackData) { ListPartsData *lpData = (ListPartsData *) callbackData; - + return simplexml_add(&(lpData->simpleXml), buffer, bufferSize); } @@ -565,11 +592,11 @@ static S3Status make_list_parts_callback(ListPartsData *lpData) S3ListPart *partDest = &(Parts[i]); ListPart *partSrc = &(lpData->parts[i]); partDest->eTag = partSrc->eTag; - partDest->partNumber = parseUnsignedInt(partSrc->partNumber); - partDest->size = parseUnsignedInt(partSrc->size); + partDest->partNumber = parseUnsignedInt(partSrc->partNumber); + partDest->size = parseUnsignedInt(partSrc->size); partDest->lastModified = parseIso8601Time(partSrc->lastModified); } - + return (*(lpData->listPartsCallback)) (isTruncated, lpData->nextPartNumberMarker, lpData->initiatorId, lpData->initiatorDisplayName, lpData->ownerId, @@ -578,7 +605,7 @@ static S3Status make_list_parts_callback(ListPartsData *lpData) } -static void listMultipartCompleteCallback(S3Status requestStatus, +static void listMultipartCompleteCallback(S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, void *callbackData) { @@ -598,7 +625,7 @@ static void listMultipartCompleteCallback(S3Status requestStatus, } -static void listPartsCompleteCallback(S3Status requestStatus, +static void listPartsCompleteCallback(S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, void *callbackData) { @@ -641,57 +668,57 @@ static S3Status listMultipartXmlCallback(const char *elementPath, } else if (!strcmp(elementPath, "ListMultipartUploadsResult/Upload/Key")) { - ListMultipartUpload *uploads = + ListMultipartUpload *uploads = &(lmData->uploads[lmData->uploadsCount]); string_buffer_append(uploads->key, data, dataLen, fit); } - else if (!strcmp(elementPath, + else if (!strcmp(elementPath, "ListMultipartUploadsResult/Upload/Initiated")) { - ListMultipartUpload *uploads = + ListMultipartUpload *uploads = &(lmData->uploads[lmData->uploadsCount]); string_buffer_append(uploads->initiated, data, dataLen, fit); } else if (!strcmp(elementPath, "ListMultipartUploadsResult/Upload/UploadId")) { - ListMultipartUpload *uploads = + ListMultipartUpload *uploads = &(lmData->uploads[lmData->uploadsCount]); string_buffer_append(uploads->uploadId, data, dataLen, fit); } else if (!strcmp(elementPath, "ListMultipartUploadsResult/Upload/Initiator/ID")) { - ListMultipartUpload *uploads = + ListMultipartUpload *uploads = &(lmData->uploads[lmData->uploadsCount]); string_buffer_append(uploads->initiatorId, data, dataLen, fit); } else if (!strcmp (elementPath, "ListMultipartUploadsResult/Upload/Initiator/DisplayName")) { - ListMultipartUpload *uploads = + ListMultipartUpload *uploads = &(lmData->uploads[lmData->uploadsCount]); string_buffer_append(uploads->initiatorDisplayName, data, dataLen, fit); } else if (!strcmp(elementPath, "ListMultipartUploadsResult/Upload/Owner/ID")) { - ListMultipartUpload *uploads = + ListMultipartUpload *uploads = &(lmData->uploads[lmData->uploadsCount]); string_buffer_append(uploads->ownerId, data, dataLen, fit); } else if (!strcmp - (elementPath, + (elementPath, "ListMultipartUploadsResult/Upload/Owner/DisplayName")) { - ListMultipartUpload *uploads = + ListMultipartUpload *uploads = &(lmData->uploads[lmData->uploadsCount]); string_buffer_append (uploads->ownerDisplayName, data, dataLen, fit); } - else if (!strcmp(elementPath, + else if (!strcmp(elementPath, "ListMultipartUploadsResult/Upload/StorageClass")) { - ListMultipartUpload *uploads = + ListMultipartUpload *uploads = &(lmData->uploads[lmData->uploadsCount]); string_buffer_append(uploads->storageClass, data, dataLen, fit); } - else if (!strcmp(elementPath, + else if (!strcmp(elementPath, "ListMultipartUploadsResult/CommonPrefixes/Prefix")) { int which = lmData->commonPrefixesCount; lmData->commonPrefixLens[which] += @@ -797,7 +824,7 @@ static S3Status listPartsXmlCallback(const char *elementPath, else if (!strcmp(elementPath, "ListPartsResult/Part/Size")) { ListPart *parts = &(lpData->parts[lpData->partsCount]); string_buffer_append(parts->size, data, dataLen, fit); - } + } } else { if (!strcmp(elementPath, "ListPartsResult/Part")) { @@ -816,7 +843,7 @@ static S3Status listPartsXmlCallback(const char *elementPath, // Initialize the next one initialize_list_part(&(lpData->parts[lpData->partsCount])); } - } + } } /* Avoid compiler error about variable set but not used */ @@ -829,15 +856,16 @@ static S3Status listPartsXmlCallback(const char *elementPath, void S3_list_multipart_uploads(S3BucketContext *bucketContext, const char *prefix, const char *keymarker, const char *uploadidmarker, - const char *encodingtype, const char *delimiter, + const char *encodingtype, const char *delimiter, int maxuploads, S3RequestContext *requestContext, + int timeoutMs, const S3ListMultipartUploadsHandler *handler, void *callbackData) { // Compose the query params string_buffer(queryParams, 4096); string_buffer_initialize(queryParams); - + #define safe_append(name, value) \ do { \ int fit; \ @@ -858,7 +886,7 @@ void S3_list_multipart_uploads(S3BucketContext *bucketContext, } \ amp = 1; \ char encoded[3 * 1024]; \ - if (!urlEncode(encoded, value, 1024)) { \ + if (!urlEncode(encoded, value, 1024, 1)) { \ (*(handler->responseHandler.completeCallback)) \ (S3StatusQueryParamsTooLong, 0, callbackData); \ return; \ @@ -871,8 +899,8 @@ void S3_list_multipart_uploads(S3BucketContext *bucketContext, return; \ } \ } while (0) - - + + int amp = 0; if (prefix && *prefix) { safe_append("prefix", prefix); @@ -895,31 +923,31 @@ void S3_list_multipart_uploads(S3BucketContext *bucketContext, maxuploads); safe_append("max-uploads", maxUploadsString); } - + ListMultipartData *lmData = (ListMultipartData *) malloc(sizeof(ListMultipartData)); - + if (!lmData) { (*(handler->responseHandler.completeCallback)) (S3StatusOutOfMemory, 0, callbackData); return; } - + simplexml_initialize(&(lmData->simpleXml), &listMultipartXmlCallback, lmData); - - lmData->responsePropertiesCallback = + + lmData->responsePropertiesCallback = handler->responseHandler.propertiesCallback; lmData->listMultipartCallback = handler->responseXmlCallback; - lmData->responseCompleteCallback = + lmData->responseCompleteCallback = handler->responseHandler.completeCallback; lmData->callbackData = callbackData; - + string_buffer_initialize(lmData->isTruncated); string_buffer_initialize(lmData->nextKeyMarker); string_buffer_initialize(lmData->nextUploadIdMarker); initialize_list_multipart_data(lmData); - + // Set up the RequestParams RequestParams params = { @@ -930,7 +958,8 @@ void S3_list_multipart_uploads(S3BucketContext *bucketContext, bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId bucketContext->secretAccessKey, // secretAccessKey - bucketContext->securityToken }, // securityToken + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion 0, // key queryParams[0] ? queryParams : 0, // queryParams "uploads", // subResource @@ -945,9 +974,10 @@ void S3_list_multipart_uploads(S3BucketContext *bucketContext, 0, // toS3CallbackTotalSize &listMultipartDataCallback, // fromS3Callback &listMultipartCompleteCallback, // completeCallback - lmData // callbackData + lmData, // callbackData + timeoutMs // timeoutMs }; - + // Perform the request request_perform(¶ms, requestContext); } @@ -957,12 +987,13 @@ void S3_list_parts(S3BucketContext *bucketContext, const char *key, const char *partnumbermarker, const char *uploadid, const char *encodingtype, int maxparts, S3RequestContext *requestContext, + int timeoutMs, const S3ListPartsHandler *handler, void *callbackData) { // Compose the query params string_buffer(queryParams, 4096); string_buffer_initialize(queryParams); - + #define safe_append(name, value) \ do { \ int fit; \ @@ -983,7 +1014,7 @@ void S3_list_parts(S3BucketContext *bucketContext, const char *key, } \ amp = 1; \ char encoded[3 * 1024]; \ - if (!urlEncode(encoded, value, 1024)) { \ + if (!urlEncode(encoded, value, 1024, 1)) { \ (*(handler->responseHandler.completeCallback)) \ (S3StatusQueryParamsTooLong, 0, callbackData); \ return; \ @@ -996,9 +1027,9 @@ void S3_list_parts(S3BucketContext *bucketContext, const char *key, return; \ } \ } while (0) - + char subResource[512]; - snprintf(subResource, 512, "uploadId=%s", uploadid); + snprintf(subResource, 512, "uploadId=%s", uploadid); int amp = 0; if (partnumbermarker && *partnumbermarker) { @@ -1012,26 +1043,26 @@ void S3_list_parts(S3BucketContext *bucketContext, const char *key, snprintf(maxPartsString, sizeof(maxPartsString), "%d", maxparts); safe_append("max-parts", maxPartsString); } - + ListPartsData *lpData = (ListPartsData *) malloc(sizeof(ListPartsData)); - + if (!lpData) { (*(handler->responseHandler.completeCallback)) (S3StatusOutOfMemory, 0, callbackData); return; } - + simplexml_initialize(&(lpData->simpleXml), &listPartsXmlCallback, lpData); - - lpData->responsePropertiesCallback = + + lpData->responsePropertiesCallback = handler->responseHandler.propertiesCallback; lpData->listPartsCallback = handler->responseXmlCallback; - lpData->responseCompleteCallback = + lpData->responseCompleteCallback = handler->responseHandler.completeCallback; lpData->callbackData = callbackData; - + string_buffer_initialize(lpData->isTruncated); string_buffer_initialize(lpData->nextPartNumberMarker); string_buffer_initialize(lpData->initiatorId); @@ -1051,7 +1082,8 @@ void S3_list_parts(S3BucketContext *bucketContext, const char *key, bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId bucketContext->secretAccessKey, // secretAccessKey - bucketContext->securityToken }, // securityToken + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion key, // key queryParams[0] ? queryParams : 0, // queryParams subResource, // subResource @@ -1066,9 +1098,10 @@ void S3_list_parts(S3BucketContext *bucketContext, const char *key, 0, // toS3CallbackTotalSize &listPartsDataCallback, // fromS3Callback &listPartsCompleteCallback, // completeCallback - lpData // callbackData + lpData, // callbackData + timeoutMs // timeoutMs }; - + // Perform the request request_perform(¶ms, requestContext); } diff --git a/src/object.c b/src/object.c index 445b067..e8df4b0 100644 --- a/src/object.c +++ b/src/object.c @@ -1,13 +1,15 @@ /** ************************************************************************** * object.c - * + * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include @@ -36,6 +42,7 @@ void S3_put_object(const S3BucketContext *bucketContext, const char *key, uint64_t contentLength, const S3PutProperties *putProperties, S3RequestContext *requestContext, + int timeoutMs, const S3PutObjectHandler *handler, void *callbackData) { // Set up the RequestParams @@ -48,7 +55,8 @@ void S3_put_object(const S3BucketContext *bucketContext, const char *key, bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId bucketContext->secretAccessKey, // secretAccessKey - bucketContext->securityToken }, // securityToken + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion key, // key 0, // queryParams 0, // subResource @@ -63,7 +71,8 @@ void S3_put_object(const S3BucketContext *bucketContext, const char *key, contentLength, // toS3CallbackTotalSize 0, // fromS3Callback handler->responseHandler.completeCallback, // completeCallback - callbackData // callbackData + callbackData, // callbackData + timeoutMs // timeoutMs }; // Perform the request @@ -86,7 +95,7 @@ typedef struct CopyObjectData int eTagReturnSize; char *eTagReturn; int eTagReturnLen; - + string_buffer(lastModified, 256); } CopyObjectData; @@ -107,7 +116,7 @@ static S3Status copyObjectXmlCallback(const char *elementPath, if (coData->eTagReturnSize && coData->eTagReturn) { coData->eTagReturnLen += snprintf(&(coData->eTagReturn[coData->eTagReturnLen]), - coData->eTagReturnSize - + coData->eTagReturnSize - coData->eTagReturnLen - 1, "%.*s", dataLen, data); if (coData->eTagReturnLen >= coData->eTagReturnSize) { @@ -128,7 +137,7 @@ static S3Status copyObjectPropertiesCallback (const S3ResponseProperties *responseProperties, void *callbackData) { CopyObjectData *coData = (CopyObjectData *) callbackData; - + return (*(coData->responsePropertiesCallback)) (responseProperties, coData->callbackData); } @@ -143,7 +152,7 @@ static S3Status copyObjectDataCallback(int bufferSize, const char *buffer, } -static void copyObjectCompleteCallback(S3Status requestStatus, +static void copyObjectCompleteCallback(S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, void *callbackData) { @@ -172,6 +181,7 @@ void S3_copy_object(const S3BucketContext *bucketContext, const char *key, const S3PutProperties *putProperties, int64_t *lastModifiedReturn, int eTagReturnSize, char *eTagReturn, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData) { /* Use the range copier with 0 length */ @@ -182,21 +192,24 @@ void S3_copy_object(const S3BucketContext *bucketContext, const char *key, putProperties, lastModifiedReturn, eTagReturnSize, eTagReturn, requestContext, + timeoutMs, handler, callbackData); } void S3_copy_object_range(const S3BucketContext *bucketContext, const char *key, - const char *destinationBucket, const char *destinationKey, - const int partNo, const char *uploadId, - const unsigned long startOffset, const unsigned long count, + const char *destinationBucket, + const char *destinationKey, const int partNo, + const char *uploadId, const unsigned long startOffset, + const unsigned long count, const S3PutProperties *putProperties, int64_t *lastModifiedReturn, int eTagReturnSize, char *eTagReturn, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData) { // Create the callback data - CopyObjectData *data = + CopyObjectData *data = (CopyObjectData *) malloc(sizeof(CopyObjectData)); if (!data) { (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData); @@ -219,11 +232,11 @@ void S3_copy_object_range(const S3BucketContext *bucketContext, const char *key, string_buffer_initialize(data->lastModified); // If there's a sequence ID > 0 then add a subResource, OTW pass in NULL - char subResource[512]; - char *subRsrc = NULL; + char queryParams[512]; + char *qp = NULL; if (partNo > 0) { - snprintf(subResource, 512, "partNumber=%d&uploadId=%s", partNo, uploadId); - subRsrc = subResource; + snprintf(queryParams, 512, "partNumber=%d&uploadId=%s", partNo, uploadId); + qp = queryParams; } // Set up the RequestParams @@ -231,16 +244,17 @@ void S3_copy_object_range(const S3BucketContext *bucketContext, const char *key, { HttpRequestTypeCOPY, // httpRequestType { bucketContext->hostName, // hostName - destinationBucket ? destinationBucket : + destinationBucket ? destinationBucket : bucketContext->bucketName, // bucketName bucketContext->protocol, // protocol bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId bucketContext->secretAccessKey, // secretAccessKey - bucketContext->securityToken }, // securityToken + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion destinationKey ? destinationKey : key, // key - 0, // queryParams - subRsrc, // subResource + qp, // queryParams + 0, // subResource bucketContext->bucketName, // copySourceBucketName key, // copySourceKey 0, // getConditions @@ -252,7 +266,8 @@ void S3_copy_object_range(const S3BucketContext *bucketContext, const char *key, 0, // toS3CallbackTotalSize ©ObjectDataCallback, // fromS3Callback ©ObjectCompleteCallback, // completeCallback - data // callbackData + data, // callbackData + timeoutMs // timeoutMs }; // Perform the request @@ -266,6 +281,7 @@ void S3_get_object(const S3BucketContext *bucketContext, const char *key, const S3GetConditions *getConditions, uint64_t startByte, uint64_t byteCount, S3RequestContext *requestContext, + int timeoutMs, const S3GetObjectHandler *handler, void *callbackData) { // Set up the RequestParams @@ -278,7 +294,8 @@ void S3_get_object(const S3BucketContext *bucketContext, const char *key, bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId bucketContext->secretAccessKey, // secretAccessKey - bucketContext->securityToken }, // securityToken + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion key, // key 0, // queryParams 0, // subResource @@ -293,7 +310,8 @@ void S3_get_object(const S3BucketContext *bucketContext, const char *key, 0, // toS3CallbackTotalSize handler->getObjectDataCallback, // fromS3Callback handler->responseHandler.completeCallback, // completeCallback - callbackData // callbackData + callbackData, // callbackData + timeoutMs // timeoutMs }; // Perform the request @@ -305,6 +323,7 @@ void S3_get_object(const S3BucketContext *bucketContext, const char *key, void S3_head_object(const S3BucketContext *bucketContext, const char *key, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData) { // Set up the RequestParams @@ -317,7 +336,8 @@ void S3_head_object(const S3BucketContext *bucketContext, const char *key, bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId bucketContext->secretAccessKey, // secretAccessKey - bucketContext->securityToken }, // securityToken + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion key, // key 0, // queryParams 0, // subResource @@ -332,18 +352,20 @@ void S3_head_object(const S3BucketContext *bucketContext, const char *key, 0, // toS3CallbackTotalSize 0, // fromS3Callback handler->completeCallback, // completeCallback - callbackData // callbackData + callbackData, // callbackData + timeoutMs // timeoutMs }; // Perform the request request_perform(¶ms, requestContext); } - + // delete object -------------------------------------------------------------- void S3_delete_object(const S3BucketContext *bucketContext, const char *key, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData) { // Set up the RequestParams @@ -356,7 +378,8 @@ void S3_delete_object(const S3BucketContext *bucketContext, const char *key, bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId bucketContext->secretAccessKey, // secretAccessKey - bucketContext->securityToken }, // securityToken + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion key, // key 0, // queryParams 0, // subResource @@ -371,7 +394,8 @@ void S3_delete_object(const S3BucketContext *bucketContext, const char *key, 0, // toS3CallbackTotalSize 0, // fromS3Callback handler->completeCallback, // completeCallback - callbackData // callbackData + callbackData, // callbackData + timeoutMs // timeoutMs }; // Perform the request diff --git a/src/request.c b/src/request.c index df871b2..dd66863 100644 --- a/src/request.c +++ b/src/request.c @@ -1,13 +1,15 @@ /** ************************************************************************** * request.c - * + * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include @@ -29,14 +35,26 @@ #include #include #include +#include #include "request.h" #include "request_context.h" #include "response_headers_handler.h" -#include "util.h" +#ifdef __APPLE__ +#include +#define S3_SHA256_DIGEST_LENGTH CC_SHA256_DIGEST_LENGTH +#else +#include +#include +#define S3_SHA256_DIGEST_LENGTH SHA256_DIGEST_LENGTH +#endif #define USER_AGENT_SIZE 256 #define REQUEST_STACK_SIZE 32 +#define SIGNATURE_SCOPE_SIZE 64 + +//#define SIGNATURE_DEBUG + static int verifyPeer; static char userAgentG[USER_AGENT_SIZE]; @@ -61,15 +79,24 @@ typedef struct RequestComputedValues // Storage for amzHeaders (the +256 is for x-amz-acl and x-amz-date) char amzHeadersRaw[COMPACTED_METADATA_BUFFER_SIZE + 256 + 1]; - // Canonicalized x-amz- headers - string_multibuffer(canonicalizedAmzHeaders, + // Length of populated data in raw buffer + int amzHeadersRawLength; + + // Canonicalized headers for signature + string_multibuffer(canonicalizedSignatureHeaders, COMPACTED_METADATA_BUFFER_SIZE + 256 + 1); + // Delimited list of header names used for signature + char signedHeaders[COMPACTED_METADATA_BUFFER_SIZE]; + // URL-Encoded key char urlEncodedKey[MAX_URLENCODED_KEY_SIZE + 1]; // Canonicalized resource - char canonicalizedResource[MAX_CANONICALIZED_RESOURCE_SIZE + 1]; + char canonicalURI[MAX_CANONICALIZED_RESOURCE_SIZE + 1]; + + // Canonical sub-resource & query string + char canonicalQueryString[MAX_CANONICALIZED_RESOURCE_SIZE + 1]; // Cache-Control header (or empty) char cacheControlHeader[128]; @@ -105,16 +132,28 @@ typedef struct RequestComputedValues char rangeHeader[128]; // Authorization header - char authorizationHeader[128]; + char authorizationHeader[4096]; + + // Request date stamp + char requestDateISO8601[64]; + + // Credential used for authorization signature + char authCredential[MAX_CREDENTIAL_SIZE + 1]; + + // Computed request signature (hex string) + char requestSignatureHex[S3_SHA256_DIGEST_LENGTH * 2 + 1]; // Host header char hostHeader[128]; + + // Hex string of hash of request payload + char payloadHash[S3_SHA256_DIGEST_LENGTH * 2 + 1]; } RequestComputedValues; // Called whenever we detect that the request headers have been completely // processed; which happens either when we get our first read/write callback, -// or the request is finished being procesed. Returns nonzero on success, +// or the request is finished being processed. Returns nonzero on success, // zero on failure. static void request_headers_done(Request *request) { @@ -127,7 +166,7 @@ static void request_headers_done(Request *request) // Get the http response code long httpResponseCode; request->httpResponseCode = 0; - if (curl_easy_getinfo(request->curl, CURLINFO_RESPONSE_CODE, + if (curl_easy_getinfo(request->curl, CURLINFO_RESPONSE_CODE, &httpResponseCode) != CURLE_OK) { // Not able to get the HTTP response code - error request->status = S3StatusInternalError; @@ -137,7 +176,7 @@ static void request_headers_done(Request *request) request->httpResponseCode = httpResponseCode; } - response_headers_handler_done(&(request->responseHeadersHandler), + response_headers_handler_done(&(request->responseHeadersHandler), request->curl); // Only make the callback if it was a successful request; otherwise we're @@ -146,7 +185,7 @@ static void request_headers_done(Request *request) (request->httpResponseCode >= 200) && (request->httpResponseCode <= 299)) { request->status = (*(request->propertiesCallback)) - (&(request->responseHeadersHandler.responseProperties), + (&(request->responseHeadersHandler.responseProperties), request->callbackData); } } @@ -186,7 +225,7 @@ static size_t curl_read_func(void *ptr, size_t size, size_t nmemb, void *data) if (!request->toS3Callback || !request->toS3CallbackBytesRemaining) { return 0; } - + // Don't tell the callback that we are willing to accept more data than we // really are if (len > request->toS3CallbackBytesRemaining) { @@ -224,7 +263,7 @@ static size_t curl_write_func(void *ptr, size_t size, size_t nmemb, } // On HTTP error, we expect to parse an HTTP error response - if ((request->httpResponseCode < 200) || + if ((request->httpResponseCode < 200) || (request->httpResponseCode > 299)) { request->status = error_parser_add (&(request->errorParser), (char *) ptr, len); @@ -244,6 +283,51 @@ static size_t curl_write_func(void *ptr, size_t size, size_t nmemb, } +static S3Status append_amz_header(RequestComputedValues *values, + int addPrefix, + const char *headerName, + const char *headerValue) +{ + int rawPos = values->amzHeadersRawLength + 1; + values->amzHeaders[values->amzHeadersCount++] = &(values->amzHeadersRaw[rawPos]); + + const char *headerStr = headerName; + + char headerNameWithPrefix[S3_MAX_METADATA_SIZE - sizeof(": v")]; + if (addPrefix) { + snprintf(headerNameWithPrefix, sizeof(headerNameWithPrefix), + S3_METADATA_HEADER_NAME_PREFIX "%s", headerName); + headerStr = headerNameWithPrefix; + } + + // Make sure the new header (plus ": " plus string terminator) will fit + // in the buffer. + if ((values->amzHeadersRawLength + strlen(headerStr) + strlen(headerValue) + + 3) >= sizeof(values->amzHeadersRaw)) { + return S3StatusMetaDataHeadersTooLong; + } + + unsigned long i = 0; + for (; i < strlen(headerStr); i++) { + values->amzHeadersRaw[rawPos++] = tolower(headerStr[i]); + } + + snprintf(&(values->amzHeadersRaw[rawPos]), 3, ": "); + rawPos += 2; + + for (i = 0; i < strlen(headerValue); i++) { + values->amzHeadersRaw[rawPos++] = headerValue[i]; + } + rawPos--; + + while (isblank(values->amzHeadersRaw[rawPos])) { + rawPos--; + } + values->amzHeadersRaw[++rawPos] = '\0'; + values->amzHeadersRawLength = rawPos; + return S3StatusOK; +} + // This function 'normalizes' all x-amz-meta headers provided in // params->requestHeaders, which means it removes all whitespace from // them such that they all look exactly like this: @@ -255,73 +339,28 @@ static size_t curl_write_func(void *ptr, size_t size, size_t nmemb, // these headers in params->amzHeaders (and also sets params->amzHeadersCount // to be the count of the total number of x-amz- headers thus created). static S3Status compose_amz_headers(const RequestParams *params, + int forceUnsignedPayload, RequestComputedValues *values) { const S3PutProperties *properties = params->putProperties; values->amzHeadersCount = 0; - values->amzHeadersRaw[0] = 0; - int len = 0; - - // Append a header to amzHeaders, trimming whitespace from the end. - // Does NOT trim whitespace from the beginning. -#define headers_append(isNewHeader, format, ...) \ - do { \ - if (isNewHeader) { \ - values->amzHeaders[values->amzHeadersCount++] = \ - &(values->amzHeadersRaw[len]); \ - } \ - len += snprintf(&(values->amzHeadersRaw[len]), \ - sizeof(values->amzHeadersRaw) - len, \ - format, __VA_ARGS__); \ - if (len >= (int) sizeof(values->amzHeadersRaw)) { \ - return S3StatusMetaDataHeadersTooLong; \ - } \ - while ((len > 0) && (values->amzHeadersRaw[len - 1] == ' ')) { \ - len--; \ - } \ - values->amzHeadersRaw[len++] = 0; \ - } while (0) - -#define header_name_tolower_copy(str, l) \ - do { \ - values->amzHeaders[values->amzHeadersCount++] = \ - &(values->amzHeadersRaw[len]); \ - if ((len + l) >= (int) sizeof(values->amzHeadersRaw)) { \ - return S3StatusMetaDataHeadersTooLong; \ - } \ - int todo = l; \ - while (todo--) { \ - if ((*(str) >= 'A') && (*(str) <= 'Z')) { \ - values->amzHeadersRaw[len++] = 'a' + (*(str) - 'A'); \ - } \ - else { \ - values->amzHeadersRaw[len++] = *(str); \ - } \ - (str)++; \ - } \ - } while (0) + values->amzHeadersRaw[0] = '\0'; + values->amzHeadersRawLength = 0; // Check and copy in the x-amz-meta headers if (properties) { int i; for (i = 0; i < properties->metaDataCount; i++) { const S3NameValue *property = &(properties->metaData[i]); - char headerName[S3_MAX_METADATA_SIZE - sizeof(": v")]; - int l = snprintf(headerName, sizeof(headerName), - S3_METADATA_HEADER_NAME_PREFIX "%s", - property->name); - char *hn = headerName; - header_name_tolower_copy(hn, l); - // Copy in the value - headers_append(0, ": %s", property->value); + append_amz_header(values, 1, property->name, property->value); } // Add the x-amz-acl header, if necessary const char *cannedAclString; switch (properties->cannedAcl) { case S3CannedAclPrivate: - cannedAclString = 0; + cannedAclString = NULL; break; case S3CannedAclPublicRead: cannedAclString = "public-read"; @@ -329,52 +368,80 @@ static S3Status compose_amz_headers(const RequestParams *params, case S3CannedAclPublicReadWrite: cannedAclString = "public-read-write"; break; + case S3CannedAclBucketOwnerFullControl: + cannedAclString = "bucket-owner-full-control"; + break; default: // S3CannedAclAuthenticatedRead cannedAclString = "authenticated-read"; break; } if (cannedAclString) { - headers_append(1, "x-amz-acl: %s", cannedAclString); + append_amz_header(values, 0, "x-amz-acl", cannedAclString); } // Add the x-amz-server-side-encryption header, if necessary if (properties->useServerSideEncryption) { - headers_append(1, "x-amz-server-side-encryption: %s", "AES256"); + append_amz_header(values, 0, "x-amz-server-side-encryption", + "AES256"); } } // Add the x-amz-date header - time_t now = time(NULL); - char date[64]; - struct tm gmt; - strftime(date, sizeof(date), "%a, %d %b %Y %H:%M:%S GMT", gmtime_r(&now, &gmt)); - headers_append(1, "x-amz-date: %s", date); + append_amz_header(values, 0, "x-amz-date", values->requestDateISO8601); if (params->httpRequestType == HttpRequestTypeCOPY) { // Add the x-amz-copy-source header - if (params->copySourceBucketName && params->copySourceBucketName[0] && - params->copySourceKey && params->copySourceKey[0]) { - headers_append(1, "x-amz-copy-source: /%s/%s", - params->copySourceBucketName, - params->copySourceKey); + if (params->copySourceBucketName && params->copySourceBucketName[0] + && params->copySourceKey && params->copySourceKey[0]) { + char bucketKey[S3_MAX_METADATA_SIZE]; + snprintf(bucketKey, sizeof(bucketKey), "/%s/%s", + params->copySourceBucketName, params->copySourceKey); + append_amz_header(values, 0, "x-amz-copy-source", bucketKey); } // If byteCount != 0 then we're just copying a range, add header if (params->byteCount > 0) { - headers_append(1, "x-amz-copy-source-range: bytes=%ld-%ld", - params->startByte, - params->startByte + params->byteCount); + char byteRange[S3_MAX_METADATA_SIZE]; + snprintf(byteRange, sizeof(byteRange), "bytes=%zd-%zd", + params->startByte, params->startByte + params->byteCount); + append_amz_header(values, 0, "x-amz-copy-source-range", byteRange); } // And the x-amz-metadata-directive header if (properties) { - headers_append(1, "%s", "x-amz-metadata-directive: REPLACE"); + append_amz_header(values, 0, "x-amz-metadata-directive", "REPLACE"); } } // Add the x-amz-security-token header if necessary if (params->bucketContext.securityToken) { - headers_append(1, "x-amz-security-token: %s", - params->bucketContext.securityToken); + append_amz_header(values, 0, "x-amz-security-token", + params->bucketContext.securityToken); + } + + if (!forceUnsignedPayload + && (params->httpRequestType == HttpRequestTypeGET + || params->httpRequestType == HttpRequestTypeCOPY + || params->httpRequestType == HttpRequestTypeDELETE + || params->httpRequestType == HttpRequestTypeHEAD)) { + // empty payload + unsigned char md[S3_SHA256_DIGEST_LENGTH]; +#ifdef __APPLE__ + CC_SHA256("", 0, md); +#else + SHA256((const unsigned char*) "", 0, md); +#endif + values->payloadHash[0] = '\0'; + int i = 0; + for (; i < S3_SHA256_DIGEST_LENGTH; i++) { + snprintf(&(values->payloadHash[i * 2]), 3, "%02x", md[i]); + } } + else { + // TODO: figure out how to manage signed payloads + strcpy(values->payloadHash, "UNSIGNED-PAYLOAD"); + } + + append_amz_header(values, 0, "x-amz-content-sha256", + values->payloadHash); return S3StatusOK; } @@ -460,14 +527,27 @@ static S3Status compose_standard_headers(const RequestParams *params, len--; } values->hostHeader[len] = 0; - } else { - values->hostHeader[0] = 0; + } + else { + size_t len = snprintf( + values->hostHeader, + sizeof(values->hostHeader), + "Host: %s", + params->bucketContext.hostName ? + params->bucketContext.hostName : defaultHostNameG); + if (len >= sizeof(values->hostHeader)) { + return S3StatusUriTooLong; + } + while (is_blank(values->hostHeader[len])) { + len--; + } + values->hostHeader[len] = 0; } // Cache-Control do_put_header("Cache-Control: %s", cacheControl, cacheControlHeader, S3StatusBadCacheControl, S3StatusCacheControlTooLong); - + // ContentType do_put_header("Content-Type: %s", contentType, contentTypeHeader, S3StatusBadContentType, S3StatusContentTypeTooLong); @@ -481,12 +561,12 @@ static S3Status compose_standard_headers(const RequestParams *params, contentDispositionFilename, contentDispositionHeader, S3StatusBadContentDispositionFilename, S3StatusContentDispositionFilenameTooLong); - + // ContentEncoding - do_put_header("Content-Encoding: %s", contentEncoding, + do_put_header("Content-Encoding: %s", contentEncoding, contentEncodingHeader, S3StatusBadContentEncoding, S3StatusContentEncodingTooLong); - + // Expires if (params->putProperties && (params->putProperties->expires >= 0)) { time_t t = (time_t) params->putProperties->expires; @@ -523,28 +603,28 @@ static S3Status compose_standard_headers(const RequestParams *params, else { values->ifUnmodifiedSinceHeader[0] = 0; } - + // If-Match header do_get_header("If-Match: %s", ifMatchETag, ifMatchHeader, S3StatusBadIfMatchETag, S3StatusIfMatchETagTooLong); - + // If-None-Match header do_get_header("If-None-Match: %s", ifNotMatchETag, ifNoneMatchHeader, - S3StatusBadIfNotMatchETag, + S3StatusBadIfNotMatchETag, S3StatusIfNotMatchETagTooLong); - + // Range header if (params->startByte || params->byteCount) { if (params->byteCount) { snprintf(values->rangeHeader, sizeof(values->rangeHeader), - "Range: bytes=%llu-%llu", + "Range: bytes=%llu-%llu", (unsigned long long) params->startByte, - (unsigned long long) (params->startByte + + (unsigned long long) (params->startByte + params->byteCount - 1)); } else { snprintf(values->rangeHeader, sizeof(values->rangeHeader), - "Range: bytes=%llu-", + "Range: bytes=%llu-", (unsigned long long) params->startByte); } } @@ -560,31 +640,32 @@ static S3Status compose_standard_headers(const RequestParams *params, static S3Status encode_key(const RequestParams *params, RequestComputedValues *values) { - return (urlEncode(values->urlEncodedKey, params->key, S3_MAX_KEY_SIZE) ? + return (urlEncode(values->urlEncodedKey, params->key, S3_MAX_KEY_SIZE, 0) ? S3StatusOK : S3StatusUriTooLong); } -// Simple comparison function for comparing two HTTP header names that are -// embedded within an HTTP header line, returning true if header1 comes -// before header2 alphabetically, false if not -static int headerle(const char *header1, const char *header2) +// Simple comparison function for comparing two "" +// delimited strings, returning true if the key of s1 comes +// before the key of s2 alphabetically, false if not +static int headerle(const char *s1, const char *s2, char delim) { while (1) { - if (*header1 == ':') { - return (*header2 != ':'); + if (*s1 == delim) { + return (*s2 != delim); } - else if (*header2 == ':') { + else if (*s2 == delim) { return 0; } - else if (*header2 < *header1) { + else if (*s2 < *s1) { return 0; } - else if (*header2 > *header1) { + else if (*s2 > *s1) { return 1; } - header1++, header2++; + s1++, s2++; } + return 0; } @@ -596,46 +677,64 @@ static int headerle(const char *header1, const char *header2) // all the string comparisons that would be done "going forward", and thus // only does the necessary string comparisons to move values back into their // sorted position. -static void header_gnome_sort(const char **headers, int size) +static void kv_gnome_sort(const char **values, int size, char delim) { int i = 0, last_highest = 0; while (i < size) { - if ((i == 0) || headerle(headers[i - 1], headers[i])) { + if ((i == 0) || headerle(values[i - 1], values[i], delim)) { i = ++last_highest; } else { - const char *tmp = headers[i]; - headers[i] = headers[i - 1]; - headers[--i] = tmp; + const char *tmp = values[i]; + values[i] = values[i - 1]; + values[--i] = tmp; } } } -// Canonicalizes the x-amz- headers into the canonicalizedAmzHeaders buffer -static void canonicalize_amz_headers(RequestComputedValues *values) +// Canonicalizes the signature headers into the canonicalizedSignatureHeaders buffer +static void canonicalize_signature_headers(RequestComputedValues *values) { // Make a copy of the headers that will be sorted - const char *sortedHeaders[S3_MAX_METADATA_COUNT]; + const char *sortedHeaders[S3_MAX_METADATA_COUNT + 3]; memcpy(sortedHeaders, values->amzHeaders, (values->amzHeadersCount * sizeof(sortedHeaders[0]))); + // add the content-type header and host header + int headerCount = values->amzHeadersCount; + if (values->contentTypeHeader[0]) { + sortedHeaders[headerCount++] = values->contentTypeHeader; + } + if (values->hostHeader[0]) { + sortedHeaders[headerCount++] = values->hostHeader; + } + if (values->rangeHeader[0]) { + sortedHeaders[headerCount++] = values->rangeHeader; + } + if (values->md5Header[0]) { + sortedHeaders[headerCount++] = values->md5Header; + } + // Now sort these - header_gnome_sort(sortedHeaders, values->amzHeadersCount); + kv_gnome_sort(sortedHeaders, headerCount, ':'); // Now copy this sorted list into the buffer, all the while: // - folding repeated headers into single lines, and // - folding multiple lines // - removing the space after the colon - int lastHeaderLen = 0, i; - char *buffer = values->canonicalizedAmzHeaders; - for (i = 0; i < values->amzHeadersCount; i++) { + int lastHeaderLen = 0; + char *buffer = values->canonicalizedSignatureHeaders; + char *hbuf = values->signedHeaders; + int i = 0; + for (; i < headerCount; i++) { const char *header = sortedHeaders[i]; const char *c = header; + char v; // If the header names are the same, append the next value - if ((i > 0) && + if ((i > 0) && !strncmp(header, sortedHeaders[i - 1], lastHeaderLen)) { // Replacing the previous newline with a comma *(buffer - 1) = ','; @@ -646,8 +745,12 @@ static void canonicalize_amz_headers(RequestComputedValues *values) else { // Copy in everything up to the space in the ": " while (*c != ' ') { - *buffer++ = *c++; + v = tolower(*c++); + *buffer++ = v; + *hbuf++ = v; } + // replace the ":" with a ";" + *(hbuf - 1) = ';'; // Save the header len since it's a new header lastHeaderLen = c - header; // Skip the space @@ -662,7 +765,7 @@ static void canonicalize_amz_headers(RequestComputedValues *values) while (is_blank(*c)) { c++; } - // Also, what has most recently been copied into buffer amy + // Also, what has most recently been copied into buffer may // have been whitespace, and since we're folding whitespace // out around this newline sequence, back buffer up over // any whitespace it contains @@ -676,6 +779,9 @@ static void canonicalize_amz_headers(RequestComputedValues *values) // Finally, add the newline *buffer++ = '\n'; } + // Remove the extra trailing semicolon from the header name list + // and terminate the string. + *(hbuf - 1) = '\0'; // Terminate the buffer *buffer = 0; @@ -683,20 +789,21 @@ static void canonicalize_amz_headers(RequestComputedValues *values) // Canonicalizes the resource into params->canonicalizedResource -static void canonicalize_resource(const char *bucketName, - const char *subResource, +static void canonicalize_resource(const S3BucketContext *context, const char *urlEncodedKey, - char *buffer) + char *buffer, unsigned int buffer_max) { int len = 0; *buffer = 0; -#define append(str) len += sprintf(&(buffer[len]), "%s", str) +#define append(str) len += snprintf(&(buffer[len]), buffer_max - len, "%s", str) - if (bucketName && bucketName[0]) { - buffer[len++] = '/'; - append(bucketName); + if (context->uriStyle == S3UriStylePath) { + if (context->bucketName && context->bucketName[0]) { + buffer[len++] = '/'; + append(context->bucketName); + } } append("/"); @@ -705,10 +812,121 @@ static void canonicalize_resource(const char *bucketName, append(urlEncodedKey); } +#undef append +} + +static void sort_query_string(const char *queryString, char *result, + unsigned int result_size) +{ +#ifdef SIGNATURE_DEBUG + printf("\n--\nsort_and_urlencode\nqueryString: %s\n", queryString); +#endif + + unsigned int numParams = 1; + const char *tmp = queryString; + while ((tmp = strchr(tmp, '&')) != NULL) { + numParams++; + tmp++; + } + + const char* params[numParams]; + + // Where did strdup go?!?? + int queryStringLen = strlen(queryString); + char *buf = (char *) malloc(queryStringLen + 1); + char *tok = buf; + strcpy(tok, queryString); + const char *token = NULL; + char *save = NULL; + unsigned int i = 0; + + while ((token = strtok_r(tok, "&", &save)) != NULL) { + tok = NULL; + params[i++] = token; + } + + kv_gnome_sort(params, numParams, '='); + +#ifdef SIGNATURE_DEBUG + for (i = 0; i < numParams; i++) { + printf("%d: %s\n", i, params[i]); + } +#endif + + // All params are urlEncoded +#define append(str) len += snprintf(&(result[len]), result_size - len, "%s", str) + unsigned int pi = 0; + unsigned int len = 0; + for (; pi < numParams; pi++) { + append(params[pi]); + append("&"); + } + // Take off the extra '&' + if (len > 0) { + result[len - 1] = 0; + } +#undef append + + free(buf); +} + + +// Canonicalize the query string part of the request into a buffer +static void canonicalize_query_string(const char *queryParams, + const char *subResource, + char *buffer, unsigned int buffer_size) +{ + int len = 0; + + *buffer = 0; + +#define append(str) len += snprintf(&(buffer[len]), buffer_size - len, "%s", str) + + if (queryParams && queryParams[0]) { + char sorted[strlen(queryParams) * 2]; + sorted[0] = '\0'; + sort_query_string(queryParams, sorted, sizeof(sorted)); + append(sorted); + } + if (subResource && subResource[0]) { - append("?"); + if (queryParams && queryParams[0]) { + append("&"); + } append(subResource); + if (!strchr(subResource, '=')) { + append("="); + } } + +#undef append +} + + +static HttpRequestType http_request_method_to_type(const char *method) +{ + if (!method) { + return HttpRequestTypeInvalid; + } + if (strcmp(method, "POST") == 0) { + return HttpRequestTypePOST; + } + else if (strcmp(method, "GET") == 0) { + return HttpRequestTypeGET; + } + else if (strcmp(method, "HEAD") == 0) { + return HttpRequestTypeHEAD; + } + else if (strcmp(method, "PUT") == 0) { + return HttpRequestTypePUT; + } + else if (strcmp(method, "COPY") == 0) { + return HttpRequestTypeCOPY; + } + else if (strcmp(method, "DELETE") == 0) { + return HttpRequestTypeDELETE; + } + return HttpRequestTypeInvalid; } @@ -735,55 +953,138 @@ static const char *http_request_type_to_verb(HttpRequestType requestType) static S3Status compose_auth_header(const RequestParams *params, RequestComputedValues *values) { - // We allow for: - // 17 bytes for HTTP-Verb + \n - // 129 bytes for Content-MD5 + \n - // 129 bytes for Content-Type + \n - // 1 byte for empty Date + \n - // CanonicalizedAmzHeaders & CanonicalizedResource - char signbuf[17 + 129 + 129 + 1 + - (sizeof(values->canonicalizedAmzHeaders) - 1) + - (sizeof(values->canonicalizedResource) - 1) + 1]; - int len = 0; - -#define signbuf_append(format, ...) \ - len += snprintf(&(signbuf[len]), sizeof(signbuf) - len, \ - format, __VA_ARGS__) + const char *httpMethod = http_request_type_to_verb(params->httpRequestType); + int canonicalRequestLen = strlen(httpMethod) + 1 + + strlen(values->canonicalURI) + 1 + + strlen(values->canonicalQueryString) + 1 + + strlen(values->canonicalizedSignatureHeaders) + 1 + + strlen(values->signedHeaders) + 1 + + 2 * S3_SHA256_DIGEST_LENGTH + 1; // 2 hex digits for each byte - signbuf_append - ("%s\n", http_request_type_to_verb(params->httpRequestType)); - - // For MD5 and Content-Type, use the value in the actual header, because - // it's already been trimmed - signbuf_append("%s\n", values->md5Header[0] ? - &(values->md5Header[sizeof("Content-MD5: ") - 1]) : ""); - - signbuf_append - ("%s\n", values->contentTypeHeader[0] ? - &(values->contentTypeHeader[sizeof("Content-Type: ") - 1]) : ""); - - signbuf_append("%s", "\n"); // Date - we always use x-amz-date + int len = 0; - signbuf_append("%s", values->canonicalizedAmzHeaders); + char canonicalRequest[canonicalRequestLen]; - signbuf_append("%s", values->canonicalizedResource); +#define buf_append(buf, format, ...) \ + len += snprintf(&(buf[len]), sizeof(buf) - len, \ + format, __VA_ARGS__) - // Generate an HMAC-SHA-1 of the signbuf - unsigned char hmac[20]; + canonicalRequest[0] = '\0'; + buf_append(canonicalRequest, "%s\n", httpMethod); + buf_append(canonicalRequest, "%s\n", values->canonicalURI); + buf_append(canonicalRequest, "%s\n", values->canonicalQueryString); + buf_append(canonicalRequest, "%s\n", values->canonicalizedSignatureHeaders); + buf_append(canonicalRequest, "%s\n", values->signedHeaders); + + buf_append(canonicalRequest, "%s", values->payloadHash); + +#ifdef SIGNATURE_DEBUG + printf("--\nCanonical Request:\n%s\n", canonicalRequest); +#endif + + len = 0; + unsigned char canonicalRequestHash[S3_SHA256_DIGEST_LENGTH]; +#ifdef __APPLE__ + CC_SHA256(canonicalRequest, strlen(canonicalRequest), canonicalRequestHash); +#else + const unsigned char *rqstData = (const unsigned char*) canonicalRequest; + SHA256(rqstData, strlen(canonicalRequest), canonicalRequestHash); +#endif + char canonicalRequestHashHex[2 * S3_SHA256_DIGEST_LENGTH + 1]; + canonicalRequestHashHex[0] = '\0'; + int i = 0; + for (; i < S3_SHA256_DIGEST_LENGTH; i++) { + buf_append(canonicalRequestHashHex, "%02x", canonicalRequestHash[i]); + } + + const char *awsRegion = S3_DEFAULT_REGION; + if (params->bucketContext.authRegion) { + awsRegion = params->bucketContext.authRegion; + } + char scope[sizeof(values->requestDateISO8601) + sizeof(awsRegion) + + sizeof("//s3/aws4_request") + 1]; + snprintf(scope, sizeof(scope), "%.8s/%s/s3/aws4_request", + values->requestDateISO8601, awsRegion); + + char stringToSign[17 + 17 + sizeof(values->requestDateISO8601) + + sizeof(scope) + sizeof(canonicalRequestHashHex) + 1]; + snprintf(stringToSign, sizeof(stringToSign), "AWS4-HMAC-SHA256\n%s\n%s\n%s", + values->requestDateISO8601, scope, canonicalRequestHashHex); + +#ifdef SIGNATURE_DEBUG + printf("--\nString to Sign:\n%s\n", stringToSign); +#endif + + const char *secretAccessKey = params->bucketContext.secretAccessKey; + char accessKey[strlen(secretAccessKey) + 5]; + snprintf(accessKey, sizeof(accessKey), "AWS4%s", secretAccessKey); + +#ifdef __APPLE__ + unsigned char dateKey[S3_SHA256_DIGEST_LENGTH]; + CCHmac(kCCHmacAlgSHA256, accessKey, strlen(accessKey), + values->requestDateISO8601, 8, dateKey); + unsigned char dateRegionKey[S3_SHA256_DIGEST_LENGTH]; + CCHmac(kCCHmacAlgSHA256, dateKey, S3_SHA256_DIGEST_LENGTH, awsRegion, + strlen(awsRegion), dateRegionKey); + unsigned char dateRegionServiceKey[S3_SHA256_DIGEST_LENGTH]; + CCHmac(kCCHmacAlgSHA256, dateRegionKey, S3_SHA256_DIGEST_LENGTH, "s3", 2, + dateRegionServiceKey); + unsigned char signingKey[S3_SHA256_DIGEST_LENGTH]; + CCHmac(kCCHmacAlgSHA256, dateRegionServiceKey, S3_SHA256_DIGEST_LENGTH, + "aws4_request", strlen("aws4_request"), signingKey); + + unsigned char finalSignature[S3_SHA256_DIGEST_LENGTH]; + CCHmac(kCCHmacAlgSHA256, signingKey, S3_SHA256_DIGEST_LENGTH, stringToSign, + strlen(stringToSign), finalSignature); +#else + const EVP_MD *sha256evp = EVP_sha256(); + unsigned char dateKey[S3_SHA256_DIGEST_LENGTH]; + HMAC(sha256evp, accessKey, strlen(accessKey), + (const unsigned char*) values->requestDateISO8601, 8, dateKey, + NULL); + unsigned char dateRegionKey[S3_SHA256_DIGEST_LENGTH]; + HMAC(sha256evp, dateKey, S3_SHA256_DIGEST_LENGTH, + (const unsigned char*) awsRegion, strlen(awsRegion), dateRegionKey, + NULL); + unsigned char dateRegionServiceKey[S3_SHA256_DIGEST_LENGTH]; + HMAC(sha256evp, dateRegionKey, S3_SHA256_DIGEST_LENGTH, + (const unsigned char*) "s3", 2, dateRegionServiceKey, NULL); + unsigned char signingKey[S3_SHA256_DIGEST_LENGTH]; + HMAC(sha256evp, dateRegionServiceKey, S3_SHA256_DIGEST_LENGTH, + (const unsigned char*) "aws4_request", strlen("aws4_request"), + signingKey, + NULL); + + unsigned char finalSignature[S3_SHA256_DIGEST_LENGTH]; + HMAC(sha256evp, signingKey, S3_SHA256_DIGEST_LENGTH, + (const unsigned char*) stringToSign, strlen(stringToSign), + finalSignature, NULL); +#endif + + len = 0; + values->requestSignatureHex[0] = '\0'; + for (i = 0; i < S3_SHA256_DIGEST_LENGTH; i++) { + buf_append(values->requestSignatureHex, "%02x", finalSignature[i]); + } + + snprintf(values->authCredential, sizeof(values->authCredential), + "%s/%.8s/%s/s3/aws4_request", params->bucketContext.accessKeyId, + values->requestDateISO8601, awsRegion); + + snprintf(values->authorizationHeader, + sizeof(values->authorizationHeader), + "Authorization: AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s", + values->authCredential, values->signedHeaders, + values->requestSignatureHex); + +#ifdef SIGNATURE_DEBUG + printf("--\nAuthorization Header:\n%s\n", values->authorizationHeader); +#endif - HMAC_SHA1(hmac, (unsigned char *) params->bucketContext.secretAccessKey, - strlen(params->bucketContext.secretAccessKey), - (unsigned char *) signbuf, len); + return S3StatusOK; - // Now base-64 encode the results - char b64[((20 + 1) * 4) / 3]; - int b64Len = base64Encode(hmac, 20, b64); - - snprintf(values->authorizationHeader, sizeof(values->authorizationHeader), - "Authorization: AWS %s:%.*s", params->bucketContext.accessKeyId, - b64Len, b64); +#undef buf_append - return S3StatusOK; } @@ -794,7 +1095,7 @@ static S3Status compose_uri(char *buffer, int bufferSize, const char *subResource, const char *queryParams) { int len = 0; - + #define uri_append(fmt, ...) \ do { \ len += snprintf(&(buffer[len]), bufferSize - len, fmt, __VA_ARGS__); \ @@ -803,13 +1104,13 @@ static S3Status compose_uri(char *buffer, int bufferSize, } \ } while (0) - uri_append("http%s://", + uri_append("http%s://", (bucketContext->protocol == S3ProtocolHTTP) ? "" : "s"); - const char *hostName = + const char *hostName = bucketContext->hostName ? bucketContext->hostName : defaultHostNameG; - if (bucketContext->bucketName && + if (bucketContext->bucketName && bucketContext->bucketName[0]) { if (bucketContext->uriStyle == S3UriStyleVirtualHost) { if (strchr(bucketContext->bucketName, '.') == NULL) { @@ -833,16 +1134,16 @@ static S3Status compose_uri(char *buffer, int bufferSize, uri_append("%s", "/"); uri_append("%s", urlEncodedKey); - + if (subResource && subResource[0]) { uri_append("?%s", subResource); } - + if (queryParams) { uri_append("%s%s", (subResource && subResource[0]) ? "&" : "?", queryParams); } - + return S3StatusOK; } @@ -861,18 +1162,18 @@ static S3Status setup_curl(Request *request, // Debugging only // curl_easy_setopt_safe(CURLOPT_VERBOSE, 1); - + // Set private data to request for the benefit of S3RequestContext curl_easy_setopt_safe(CURLOPT_PRIVATE, request); - + // Set header callback and data curl_easy_setopt_safe(CURLOPT_HEADERDATA, request); curl_easy_setopt_safe(CURLOPT_HEADERFUNCTION, &curl_header_func); - + // Set read callback, data, and readSize curl_easy_setopt_safe(CURLOPT_READFUNCTION, &curl_read_func); curl_easy_setopt_safe(CURLOPT_READDATA, request); - + // Set write callback and data curl_easy_setopt_safe(CURLOPT_WRITEFUNCTION, &curl_write_func); curl_easy_setopt_safe(CURLOPT_WRITEDATA, request); @@ -898,7 +1199,7 @@ static S3Status setup_curl(Request *request, // I think this is useful - we don't need interactive performance, we need // to complete large operations quickly curl_easy_setopt_safe(CURLOPT_TCP_NODELAY, 1); - + // Don't use Curl's 'netrc' feature curl_easy_setopt_safe(CURLOPT_NETRC, CURL_NETRC_IGNORED); @@ -922,6 +1223,12 @@ static S3Status setup_curl(Request *request, curl_easy_setopt_safe(CURLOPT_LOW_SPEED_LIMIT, 1024); curl_easy_setopt_safe(CURLOPT_LOW_SPEED_TIME, 15); + + if (params->timeoutMs > 0) { + curl_easy_setopt_safe(CURLOPT_TIMEOUT_MS, params->timeoutMs); + } + + // Append standard headers #define append_standard_header(fieldName) \ if (values-> fieldName [0]) { \ @@ -936,14 +1243,14 @@ static S3Status setup_curl(Request *request, snprintf(header, sizeof(header), "Content-Length: %llu", (unsigned long long) params->toS3CallbackTotalSize); request->headers = curl_slist_append(request->headers, header); - request->headers = curl_slist_append(request->headers, + request->headers = curl_slist_append(request->headers, "Transfer-Encoding:"); } else if (params->httpRequestType == HttpRequestTypeCOPY) { - request->headers = curl_slist_append(request->headers, + request->headers = curl_slist_append(request->headers, "Transfer-Encoding:"); } - + append_standard_header(hostHeader); append_standard_header(cacheControlHeader); append_standard_header(contentTypeHeader); @@ -961,7 +1268,7 @@ static S3Status setup_curl(Request *request, // Append x-amz- headers int i; for (i = 0; i < values->amzHeadersCount; i++) { - request->headers = + request->headers = curl_slist_append(request->headers, values->amzHeaders[i]); } @@ -991,7 +1298,7 @@ static S3Status setup_curl(Request *request, default: // HttpRequestTypeGET break; } - + return S3StatusOK; } @@ -1001,7 +1308,7 @@ static void request_deinitialize(Request *request) if (request->headers) { curl_slist_free_all(request->headers); } - + error_parser_deinitialize(&(request->errorParser)); // curl_easy_reset prevents connections from being re-used for some @@ -1012,12 +1319,13 @@ static void request_deinitialize(Request *request) } -static S3Status request_get(const RequestParams *params, +static S3Status request_get(const RequestParams *params, const RequestComputedValues *values, + const S3RequestContext *context, Request **reqReturn) { Request *request = 0; - + // Try to get one from the request stack. We hold the lock for the // shortest time possible here. pthread_mutex_lock(&requestStackMutexG); @@ -1025,7 +1333,7 @@ static S3Status request_get(const RequestParams *params, if (requestStackCountG) { request = requestStackG[--requestStackCountG]; } - + pthread_mutex_unlock(&requestStackMutexG); // If we got one, deinitialize it for re-use @@ -1052,13 +1360,13 @@ static S3Status request_get(const RequestParams *params, request->status = S3StatusOK; S3Status status; - + // Start out with no headers request->headers = 0; // Compute the URL if ((status = compose_uri - (request->uri, sizeof(request->uri), + (request->uri, sizeof(request->uri), &(params->bucketContext), values->urlEncodedKey, params->subResource, params->queryParams)) != S3StatusOK) { curl_easy_cleanup(request->curl); @@ -1073,6 +1381,15 @@ static S3Status request_get(const RequestParams *params, return status; } + if (context && context->setupCurlCallback && + (status = context->setupCurlCallback( + context->curlm, request->curl, + context->setupCurlCallbackData)) != S3StatusOK) { + curl_easy_cleanup(request->curl); + free(request); + return status; + } + request->propertiesCallback = params->propertiesCallback; request->toS3Callback = params->toS3Callback; @@ -1088,11 +1405,11 @@ static S3Status request_get(const RequestParams *params, response_headers_handler_initialize(&(request->responseHeadersHandler)); request->propertiesCallbackMade = 0; - + error_parser_initialize(&(request->errorParser)); *reqReturn = request; - + return S3StatusOK; } @@ -1128,7 +1445,7 @@ static void request_release(Request *request) S3Status request_api_initialize(const char *userAgentInfo, int flags, const char *defaultHostName) { - if (curl_global_init(CURL_GLOBAL_ALL & + if (curl_global_init(CURL_GLOBAL_ALL & ~((flags & S3_INIT_WINSOCK) ? 0 : CURL_GLOBAL_WIN32)) != CURLE_OK) { return S3StatusInternalError; @@ -1139,7 +1456,7 @@ S3Status request_api_initialize(const char *userAgentInfo, int flags, defaultHostName = S3_DEFAULT_HOSTNAME; } - if (snprintf(defaultHostNameG, S3_MAX_HOSTNAME_SIZE, + if (snprintf(defaultHostNameG, S3_MAX_HOSTNAME_SIZE, "%s", defaultHostName) >= S3_MAX_HOSTNAME_SIZE) { return S3StatusUriTooLong; } @@ -1152,20 +1469,21 @@ S3Status request_api_initialize(const char *userAgentInfo, int flags, userAgentInfo = "Unknown"; } - char platform[96]; struct utsname utsn; + char platform[sizeof(utsn.sysname) + 1 + sizeof(utsn.machine) + 1]; if (uname(&utsn)) { snprintf(platform, sizeof(platform), "Unknown"); } else { - snprintf(platform, sizeof(platform), "%s%s%s", utsn.sysname, + snprintf(platform, sizeof(platform), "%s%s%s", utsn.sysname, utsn.machine[0] ? " " : "", utsn.machine); } - snprintf(userAgentG, sizeof(userAgentG), + snprintf(userAgentG, sizeof(userAgentG), "Mozilla/4.0 (Compatible; %s; libs3 %s.%s; %s)", userAgentInfo, LIBS3_VER_MAJOR, LIBS3_VER_MINOR, platform); - + + xmlInitParser(); return S3StatusOK; } @@ -1174,64 +1492,95 @@ void request_api_deinitialize() { pthread_mutex_destroy(&requestStackMutexG); + xmlCleanupParser(); while (requestStackCountG--) { request_destroy(requestStackG[requestStackCountG]); } } -void request_perform(const RequestParams *params, S3RequestContext *context) +static S3Status setup_request(const RequestParams *params, + RequestComputedValues *computed, + int forceUnsignedPayload) { - Request *request; S3Status status; - int verifyPeerRequest = verifyPeer; - CURLcode curlstatus; - -#define return_status(status) \ - (*(params->completeCallback))(status, 0, params->callbackData); \ - return - - // These will hold the computed values - RequestComputedValues computed; // Validate the bucket name - if (params->bucketContext.bucketName && - ((status = S3_validate_bucket_name - (params->bucketContext.bucketName, - params->bucketContext.uriStyle)) != S3StatusOK)) { - return_status(status); + if (params->bucketContext.bucketName + && ((status = S3_validate_bucket_name(params->bucketContext.bucketName, + params->bucketContext.uriStyle)) + != S3StatusOK)) { + return status; } + time_t now = time(NULL); + struct tm gmt; + gmtime_r(&now, &gmt); + strftime(computed->requestDateISO8601, sizeof(computed->requestDateISO8601), + "%Y%m%dT%H%M%SZ", &gmt); + // Compose the amz headers - if ((status = compose_amz_headers(params, &computed)) != S3StatusOK) { - return_status(status); + if ((status = compose_amz_headers(params, forceUnsignedPayload, computed)) + != S3StatusOK) { + return status; } // Compose standard headers - if ((status = compose_standard_headers - (params, &computed)) != S3StatusOK) { - return_status(status); + if ((status = compose_standard_headers(params, computed)) != S3StatusOK) { + return status; } // URL encode the key - if ((status = encode_key(params, &computed)) != S3StatusOK) { - return_status(status); + if ((status = encode_key(params, computed)) != S3StatusOK) { + return status; } // Compute the canonicalized amz headers - canonicalize_amz_headers(&computed); + canonicalize_signature_headers(computed); // Compute the canonicalized resource - canonicalize_resource(params->bucketContext.bucketName, - params->subResource, computed.urlEncodedKey, - computed.canonicalizedResource); + canonicalize_resource(¶ms->bucketContext, computed->urlEncodedKey, + computed->canonicalURI, + sizeof(computed->canonicalURI)); + canonicalize_query_string(params->queryParams, params->subResource, + computed->canonicalQueryString, + sizeof(computed->canonicalQueryString)); // Compose Authorization header - if ((status = compose_auth_header(params, &computed)) != S3StatusOK) { + if ((status = compose_auth_header(params, computed)) != S3StatusOK) { + return status; + } + +#ifdef SIGNATURE_DEBUG + int i = 0; + printf("\n--\nAMZ Headers:\n"); + for (; i < computed->amzHeadersCount; i++) { + printf("%s\n", computed->amzHeaders[i]); + } +#endif + + return status; +} + +void request_perform(const RequestParams *params, S3RequestContext *context) +{ + Request *request; + S3Status status; + int verifyPeerRequest = verifyPeer; + CURLcode curlstatus; + +#define return_status(status) \ + (*(params->completeCallback))(status, 0, params->callbackData); \ + return + + // These will hold the computed values + RequestComputedValues computed; + + if ((status = setup_request(params, &computed, 0)) != S3StatusOK) { return_status(status); } - + // Get an initialized Request structure now - if ((status = request_get(params, &computed, &request)) != S3StatusOK) { + if ((status = request_get(params, &computed, context, &request)) != S3StatusOK) { return_status(status); } if (context && context->verifyPeerSet) { @@ -1239,12 +1588,12 @@ void request_perform(const RequestParams *params, S3RequestContext *context) } // Allow per-context override of verifyPeer if (verifyPeerRequest != verifyPeer) { - if ((curlstatus = curl_easy_setopt(request->curl, - CURLOPT_SSL_VERIFYPEER, - context->verifyPeer)) - != CURLE_OK) { - return_status(S3StatusFailedToInitializeRequest); - } + if ((curlstatus = curl_easy_setopt(request->curl, + CURLOPT_SSL_VERIFYPEER, + context->verifyPeer)) + != CURLE_OK) { + return_status(S3StatusFailedToInitializeRequest); + } } // If a RequestContext was provided, add the request to the curl multi @@ -1288,11 +1637,11 @@ void request_finish(Request *request) // If we haven't detected this already, we now know that the headers are // definitely done being read in request_headers_done(request); - + // If there was no error processing the request, then possibly there was // an S3 error parsed, which should be converted into the request status if (request->status == S3StatusOK) { - error_parser_convert_status(&(request->errorParser), + error_parser_convert_status(&(request->errorParser), &(request->status)); // If there still was no error recorded, then it is possible that // there was in fact an error but that there was no error XML @@ -1318,7 +1667,7 @@ void request_finish(Request *request) case 400: request->status = S3StatusHttpErrorBadRequest; break; - case 403: + case 403: request->status = S3StatusHttpErrorForbidden; break; case 404: @@ -1375,11 +1724,17 @@ S3Status request_curl_code_to_status(CURLcode code) return S3StatusFailedToConnect; case CURLE_WRITE_ERROR: case CURLE_OPERATION_TIMEDOUT: - return S3StatusConnectionFailed; + return S3StatusErrorRequestTimeout; case CURLE_PARTIAL_FILE: return S3StatusOK; +#if LIBCURL_VERSION_NUM >= 0x071101 /* 7.17.1 */ case CURLE_PEER_FAILED_VERIFICATION: +#else + case CURLE_SSL_PEER_CERTIFICATE: +#endif +#if LIBCURL_VERSION_NUM < 0x073e00 case CURLE_SSL_CACERT: +#endif return S3StatusServerFailedVerification; default: return S3StatusInternalError; @@ -1389,11 +1744,12 @@ S3Status request_curl_code_to_status(CURLcode code) S3Status S3_generate_authenticated_query_string (char *buffer, const S3BucketContext *bucketContext, - const char *key, int64_t expires, const char *resource) + const char *key, int expires, const char *resource, + const char *httpMethod) { -#define MAX_EXPIRES (((int64_t) 1 << 31) - 1) - // S3 seems to only accept expiration dates up to the number of seconds - // representably by a signed 32-bit integer + // maximum expiration period is seven days (in seconds) +#define MAX_EXPIRES 604800 + if (expires < 0) { expires = MAX_EXPIRES; } @@ -1401,68 +1757,36 @@ S3Status S3_generate_authenticated_query_string expires = MAX_EXPIRES; } - // xxx todo: rework this so that it can be incorporated into shared code - // with request_perform(). It's really unfortunate that this code is not - // shared with request_perform(). + RequestParams params = + { http_request_method_to_type(httpMethod), *bucketContext, key, NULL, + resource, + NULL, NULL, NULL, 0, 0, NULL, NULL, NULL, 0, NULL, NULL, NULL, 0}; - // URL encode the key - char urlEncodedKey[S3_MAX_KEY_SIZE * 3]; - if (key) { - urlEncode(urlEncodedKey, key, strlen(key)); + RequestComputedValues computed; + S3Status status = setup_request(¶ms, &computed, 1); + if (status != S3StatusOK) { + return status; } - else { - urlEncodedKey[0] = 0; - } - - // Compute canonicalized resource - char canonicalizedResource[MAX_CANONICALIZED_RESOURCE_SIZE]; - canonicalize_resource(bucketContext->bucketName, resource, urlEncodedKey, - canonicalizedResource); - - // We allow for: - // 17 bytes for HTTP-Verb + \n - // 1 byte for empty Content-MD5 + \n - // 1 byte for empty Content-Type + \n - // 20 bytes for Expires + \n - // 0 bytes for CanonicalizedAmzHeaders - // CanonicalizedResource - char signbuf[17 + 1 + 1 + 1 + 20 + sizeof(canonicalizedResource) + 1]; - int len = 0; - -#define signbuf_append(format, ...) \ - len += snprintf(&(signbuf[len]), sizeof(signbuf) - len, \ - format, __VA_ARGS__) - - signbuf_append("%s\n", "GET"); // HTTP-Verb - signbuf_append("%s\n", ""); // Content-MD5 - signbuf_append("%s\n", ""); // Content-Type - signbuf_append("%llu\n", (unsigned long long) expires); - signbuf_append("%s", canonicalizedResource); - - // Generate an HMAC-SHA-1 of the signbuf - unsigned char hmac[20]; - - HMAC_SHA1(hmac, (unsigned char *) bucketContext->secretAccessKey, - strlen(bucketContext->secretAccessKey), - (unsigned char *) signbuf, len); - - // Now base-64 encode the results - char b64[((20 + 1) * 4) / 3]; - int b64Len = base64Encode(hmac, 20, b64); - - // Now urlEncode that - char signature[sizeof(b64) * 3]; - urlEncode(signature, b64, b64Len); - - // Finally, compose the uri, with params: - // ?AWSAccessKeyId=xxx[&Expires=]&Signature=xxx - char queryParams[sizeof("AWSAccessKeyId=") + 20 + - sizeof("&Expires=") + 20 + - sizeof("&Signature=") + sizeof(signature) + 1]; - sprintf(queryParams, "AWSAccessKeyId=%s&Expires=%ld&Signature=%s", - bucketContext->accessKeyId, (long) expires, signature); + // Finally, compose the URI, with params + char queryParams[sizeof("X-Amz-Algorithm=AWS4-HMAC-SHA256") + + sizeof("&X-Amz-Credential=") + + sizeof(computed.authCredential) + + sizeof("&X-Amz-Date=") + + sizeof(computed.requestDateISO8601) + + sizeof("&X-Amz-Expires=") + 64 + + sizeof("&X-Amz-SignedHeaders=") + + sizeof(computed.signedHeaders) + + sizeof("&X-Amz-Signature=") + + sizeof(computed.requestSignatureHex) + 1]; + snprintf(queryParams, sizeof(queryParams), + "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=%s" + "&X-Amz-Date=%s&X-Amz-Expires=%d" + "&X-Amz-SignedHeaders=%s&X-Amz-Signature=%s", + computed.authCredential, computed.requestDateISO8601, expires, + computed.signedHeaders, computed.requestSignatureHex); return compose_uri(buffer, S3_MAX_AUTHENTICATED_QUERY_STRING_SIZE, - bucketContext, urlEncodedKey, resource, queryParams); + bucketContext, computed.urlEncodedKey, resource, + queryParams); } diff --git a/src/request_context.c b/src/request_context.c index d65a07e..345096b 100644 --- a/src/request_context.c +++ b/src/request_context.c @@ -2,12 +2,14 @@ * request_context.c * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include @@ -31,7 +37,10 @@ #include "request_context.h" -S3Status S3_create_request_context(S3RequestContext **requestContextReturn) +S3Status S3_create_request_context_ex(S3RequestContext **requestContextReturn, + CURLM *curlm, + S3SetupCurlCallback setupCurlCallback, + void *setupCurlCallbackData) { *requestContextReturn = (S3RequestContext *) malloc(sizeof(S3RequestContext)); @@ -40,34 +49,53 @@ S3Status S3_create_request_context(S3RequestContext **requestContextReturn) return S3StatusOutOfMemory; } - if (!((*requestContextReturn)->curlm = curl_multi_init())) { - free(*requestContextReturn); - return S3StatusOutOfMemory; + if (curlm) { + (*requestContextReturn)->curlm = curlm; + (*requestContextReturn)->curl_mode = S3CurlModeMultiSocket; + } + else { + if (!((*requestContextReturn)->curlm = curl_multi_init())) { + free(*requestContextReturn); + return S3StatusOutOfMemory; + } + + (*requestContextReturn)->curl_mode = S3CurlModeMultiPerform; } (*requestContextReturn)->requests = 0; (*requestContextReturn)->verifyPeer = 0; (*requestContextReturn)->verifyPeerSet = 0; + (*requestContextReturn)->setupCurlCallback = setupCurlCallback; + (*requestContextReturn)->setupCurlCallbackData = setupCurlCallbackData; return S3StatusOK; } -void S3_destroy_request_context(S3RequestContext *requestContext) +S3Status S3_create_request_context(S3RequestContext **requestContextReturn) { - curl_multi_cleanup(requestContext->curlm); + return S3_create_request_context_ex(requestContextReturn, NULL, NULL, NULL); +} + - // For each request in the context, call back its done method with - // 'interrupted' status +void S3_destroy_request_context(S3RequestContext *requestContext) +{ + // For each request in the context, remove curl handle, call back its done + // method with 'interrupted' status Request *r = requestContext->requests, *rFirst = r; if (r) do { r->status = S3StatusInterrupted; + // remove easy handle from a multi session + curl_multi_remove_handle(requestContext->curlm, r->curl); Request *rNext = r->next; request_finish(r); r = rNext; } while (r != rFirst); + if (requestContext->curl_mode == S3CurlModeMultiPerform) + curl_multi_cleanup(requestContext->curlm); + free(requestContext); } @@ -107,10 +135,62 @@ S3Status S3_runall_request_context(S3RequestContext *requestContext) } +static S3Status process_request_context(S3RequestContext *requestContext, int *retry) +{ + CURLMsg *msg; + int junk; + + *retry = 0; + + while ((msg = curl_multi_info_read(requestContext->curlm, &junk))) { + if (msg->msg != CURLMSG_DONE) { + return S3StatusInternalError; + } + Request *request; + if (curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, + (char **) (char *) &request) != CURLE_OK) { + return S3StatusInternalError; + } + // Remove the request from the list of requests + if (request->prev == request->next) { + // It was the only one on the list + requestContext->requests = 0; + } + else { + // It doesn't matter what the order of them are, so just in + // case request was at the head of the list, put the one after + // request to the head of the list + requestContext->requests = request->next; + request->prev->next = request->next; + request->next->prev = request->prev; + } + if ((msg->data.result != CURLE_OK) && + (request->status == S3StatusOK)) { + request->status = request_curl_code_to_status( + msg->data.result); + } + if (curl_multi_remove_handle(requestContext->curlm, + msg->easy_handle) != CURLM_OK) { + return S3StatusInternalError; + } + // Finish the request, ensuring that all callbacks have been made, + // and also releases the request + request_finish(request); + // Now, since a callback was made, there may be new requests + // queued up to be performed immediately, so do so + *retry = 1; + } + + return S3StatusOK; +} + + S3Status S3_runonce_request_context(S3RequestContext *requestContext, int *requestsRemainingReturn) { + S3Status s3_status; CURLMcode status; + int retry; do { status = curl_multi_perform(requestContext->curlm, @@ -126,51 +206,24 @@ S3Status S3_runonce_request_context(S3RequestContext *requestContext, return S3StatusInternalError; } - CURLMsg *msg; - int junk; - while ((msg = curl_multi_info_read(requestContext->curlm, &junk))) { - if (msg->msg != CURLMSG_DONE) { - return S3StatusInternalError; - } - Request *request; - if (curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, - (char **) (char *) &request) != CURLE_OK) { - return S3StatusInternalError; - } - // Remove the request from the list of requests - if (request->prev == request->next) { - // It was the only one on the list - requestContext->requests = 0; - } - else { - // It doesn't matter what the order of them are, so just in - // case request was at the head of the list, put the one after - // request to the head of the list - requestContext->requests = request->next; - request->prev->next = request->next; - request->next->prev = request->prev; - } - if ((msg->data.result != CURLE_OK) && - (request->status == S3StatusOK)) { - request->status = request_curl_code_to_status - (msg->data.result); - } - if (curl_multi_remove_handle(requestContext->curlm, - msg->easy_handle) != CURLM_OK) { - return S3StatusInternalError; - } - // Finish the request, ensuring that all callbacks have been made, - // and also releases the request - request_finish(request); - // Now, since a callback was made, there may be new requests - // queued up to be performed immediately, so do so - status = CURLM_CALL_MULTI_PERFORM; - } - } while (status == CURLM_CALL_MULTI_PERFORM); + s3_status = process_request_context(requestContext, &retry); + } while (s3_status == S3StatusOK && + (status == CURLM_CALL_MULTI_PERFORM || retry)); - return S3StatusOK; + return s3_status; } + +S3Status S3_process_request_context(S3RequestContext *requestContext) +{ + int retry; + /* In curl_multi_socket_action mode any new requests created during + the following call will have already started associated socket + operations, so no need to retry here */ + return process_request_context(requestContext, &retry); +} + + S3Status S3_get_request_context_fdsets(S3RequestContext *requestContext, fd_set *readFdSet, fd_set *writeFdSet, fd_set *exceptFdSet, int *maxFd) diff --git a/src/response_headers_handler.c b/src/response_headers_handler.c index c5b5084..82b8f10 100644 --- a/src/response_headers_handler.c +++ b/src/response_headers_handler.c @@ -2,12 +2,14 @@ * response_headers_handler.c * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,10 +24,15 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include #include +#include #include "response_headers_handler.h" @@ -113,44 +120,45 @@ void response_headers_handler_add(ResponseHeadersHandler *handler, int valuelen = (end - c) + 1, fit; - if (!strncmp(header, "x-amz-request-id", namelen)) { + if (!strncasecmp(header, "x-amz-request-id", namelen)) { responseProperties->requestId = string_multibuffer_current(handler->responsePropertyStrings); string_multibuffer_add(handler->responsePropertyStrings, c, valuelen, fit); } - else if (!strncmp(header, "x-amz-id-2", namelen)) { + else if (!strncasecmp(header, "x-amz-id-2", namelen)) { responseProperties->requestId2 = string_multibuffer_current(handler->responsePropertyStrings); string_multibuffer_add(handler->responsePropertyStrings, c, valuelen, fit); } - else if (!strncmp(header, "Content-Type", namelen)) { + else if (!strncasecmp(header, "Content-Type", namelen)) { responseProperties->contentType = string_multibuffer_current(handler->responsePropertyStrings); string_multibuffer_add(handler->responsePropertyStrings, c, valuelen, fit); } - else if (!strncmp(header, "Content-Length", namelen)) { + else if (!strncasecmp(header, "Content-Length", namelen)) { handler->responseProperties.contentLength = 0; while (*c) { handler->responseProperties.contentLength *= 10; handler->responseProperties.contentLength += (*c++ - '0'); } } - else if (!strncmp(header, "Server", namelen)) { + else if (!strncasecmp(header, "Server", namelen)) { responseProperties->server = string_multibuffer_current(handler->responsePropertyStrings); string_multibuffer_add(handler->responsePropertyStrings, c, valuelen, fit); } - else if (!strncmp(header, "ETag", namelen)) { + else if ((!strncasecmp(header, "ETag", namelen)) + || (!strncasecmp(header, "Etag", namelen))) { // some servers reply with Etag header responseProperties->eTag = string_multibuffer_current(handler->responsePropertyStrings); string_multibuffer_add(handler->responsePropertyStrings, c, valuelen, fit); } - else if (!strncmp(header, S3_METADATA_HEADER_NAME_PREFIX, + else if (!strncasecmp(header, S3_METADATA_HEADER_NAME_PREFIX, sizeof(S3_METADATA_HEADER_NAME_PREFIX) - 1)) { // Make sure there is room for another x-amz-meta header if (handler->responseProperties.metaDataCount == @@ -189,7 +197,7 @@ void response_headers_handler_add(ResponseHeadersHandler *handler, metaHeader->name = copiedName; metaHeader->value = copiedValue; } - else if (!strncmp(header, "x-amz-server-side-encryption", namelen)) { + else if (!strncasecmp(header, "x-amz-server-side-encryption", namelen)) { if (!strncmp(c, "AES256", sizeof("AES256") - 1)) { responseProperties->usesServerSideEncryption = 1; } diff --git a/src/s3.c b/src/s3.c index 0493b31..4343c9e 100644 --- a/src/s3.c +++ b/src/s3.c @@ -1,13 +1,15 @@ /** ************************************************************************** * s3.c - * + * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ /** @@ -63,7 +69,9 @@ static int showResponsePropertiesG = 0; static S3Protocol protocolG = S3ProtocolHTTPS; static S3UriStyle uriStyleG = S3UriStylePath; static int retriesG = 5; +static int timeoutMsG = 0; static int verifyPeerG = 0; +static const char *awsRegionG = NULL; // Environment variables, saved as globals ---------------------------------- @@ -156,6 +164,8 @@ static char putenvBufG[256]; #define TARGET_BUCKET_PREFIX_LEN (sizeof(TARGET_BUCKET_PREFIX) - 1) #define TARGET_PREFIX_PREFIX "targetPrefix=" #define TARGET_PREFIX_PREFIX_LEN (sizeof(TARGET_PREFIX_PREFIX) - 1) +#define HTTP_METHOD_PREFIX "method=" +#define HTTP_METHOD_PREFIX_LEN (sizeof(HTTP_METHOD_PREFIX) - 1) // util ---------------------------------------------------------------------- @@ -164,10 +174,10 @@ static void S3_init() { S3Status status; const char *hostname = getenv("S3_HOSTNAME"); - + if ((status = S3_initialize("s3", verifyPeerG|S3_INIT_ALL, hostname)) != S3StatusOK) { - fprintf(stderr, "Failed to initialize libs3: %s\n", + fprintf(stderr, "Failed to initialize libs3: %s\n", S3_get_status_name(status)); exit(-1); } @@ -200,14 +210,17 @@ static void usageExit(FILE *out) " -s/--show-properties : show response properties on stdout\n" " -r/--retries : retry retryable failures this number of times\n" " (default is 5)\n" +" -t/--timeout : request timeout, milliseconds. 0 if waiting forever\n" +" (default is 0)\n" " -v/--verify-peer : verify peer SSL certificate (default is no)\n" +" -g/--region : use for request authorization\n" "\n" " Environment:\n" "\n" " S3_ACCESS_KEY_ID : S3 access key ID (required)\n" " S3_SECRET_ACCESS_KEY : S3 secret access key (required)\n" " S3_HOSTNAME : specify alternative S3 host (optional)\n" -"\n" +"\n" " Commands (with and [optional parameters]) :\n" "\n" " (NOTE: all command parameters take a value and are specified using the\n" @@ -244,17 +257,24 @@ static void usageExit(FILE *out) " setacl : Set the ACL of a bucket or key\n" " [/] : Bucket or bucket/key to set the ACL of\n" " [filename] : Input filename for ACL (default is stdin)\n" +" getlifecycle : Get the lifecycle of a bucket\n" +" : Bucket or bucket to get the lifecycle of\n" +" [filename] : Output filename for lifecycle (default is stdout)\n" +"\n" +" setlifecycle : Set the lifecycle of a bucket or key\n" +" : Bucket or bucket to set the lifecycle of\n" +" [filename] : Input filename for lifecycle (default is stdin)\n" "\n" " getlogging : Get the logging status of a bucket\n" " : Bucket to get the logging status of\n" -" [filename] : Output filename for ACL (default is stdout)\n" +" [filename] : Output filename for logging (default is stdout)\n" "\n" " setlogging : Set the logging status of a bucket\n" " : Bucket to set the logging status of\n" " [targetBucket] : Target bucket to log to; if not present, disables\n" " logging\n" " [targetPrefix] : Key prefix to use for logs\n" -" [filename] : Input filename for ACL (default is stdin)\n" +" [filename] : Input filename for logging (default is stdin)\n" "\n" " put : Puts an object\n" " / : Bucket/key to put object to\n" @@ -322,6 +342,8 @@ static void usageExit(FILE *out) " [expires] : Expiration date for query string\n" " [resource] : Sub-resource of key for query string, without a\n" " leading '?', for example, \"torrent\"\n" +" [method] : HTTP method for use with the query string\n" +" : (default is \"GET\")" "\n" " listmultiparts : Show multipart uploads\n" " : Bucket multipart uploads belongs to\n" @@ -383,7 +405,7 @@ static uint64_t convertInt(const char *str, const char *paramName) while (*str) { if (!isdigit(*str)) { - fprintf(stderr, "\nERROR: Nondigit in %s parameter: %c\n", + fprintf(stderr, "\nERROR: Nondigit in %s parameter: %c\n", paramName, *str); usageExit(stderr); } @@ -410,7 +432,7 @@ typedef struct growbuffer // returns nonzero on success, zero on out of memory static int growbuffer_append(growbuffer **gb, const char *data, int dataLen) { - int toCopy = 0 ; + int origDataLen = dataLen; while (dataLen) { growbuffer *buf = *gb ? (*gb)->prev : 0; if (!buf || (buf->size == sizeof(buf->data))) { @@ -432,21 +454,21 @@ static int growbuffer_append(growbuffer **gb, const char *data, int dataLen) } } - toCopy = (sizeof(buf->data) - buf->size); + int toCopy = (sizeof(buf->data) - buf->size); if (toCopy > dataLen) { toCopy = dataLen; } memcpy(&(buf->data[buf->size]), data, toCopy); - + buf->size += toCopy, data += toCopy, dataLen -= toCopy; } - return toCopy; + return origDataLen; } -static void growbuffer_read(growbuffer **gb, int amt, int *amtReturn, +static void growbuffer_read(growbuffer **gb, int amt, int *amtReturn, char *buffer) { *amtReturn = 0; @@ -460,7 +482,7 @@ static void growbuffer_read(growbuffer **gb, int amt, int *amtReturn, *amtReturn = (buf->size > amt) ? amt : buf->size; memcpy(buffer, &(buf->data[buf->start]), *amtReturn); - + buf->start += *amtReturn, buf->size -= *amtReturn; if (buf->size == 0) { @@ -473,6 +495,7 @@ static void growbuffer_read(growbuffer **gb, int amt, int *amtReturn, buf->next->prev = buf->prev; } free(buf); + buf = NULL; } } @@ -567,7 +590,7 @@ static int64_t parseIso8601Time(const char *str) str++; } } - + if (checkString(str, "-dd:dd") || checkString(str, "+dd:dd")) { int sign = (*str++ == '-') ? -1 : 1; int hours = nextnum(); @@ -608,7 +631,7 @@ static int convert_simple_acl(char *aclXml, char *ownerId, return 0; \ } \ } while (0) - + #define COPY_STRING_MAXLEN(field, maxlen) \ do { \ SKIP_SPACE(1); \ @@ -628,7 +651,7 @@ static int convert_simple_acl(char *aclXml, char *ownerId, if (!*aclXml) { break; } - + // Skip Type lines and dash lines if (!strncmp(aclXml, "Type", sizeof("Type") - 1) || (*aclXml == '-')) { @@ -637,7 +660,7 @@ static int convert_simple_acl(char *aclXml, char *ownerId, } continue; } - + if (!strncmp(aclXml, "OwnerID", sizeof("OwnerID") - 1)) { aclXml += sizeof("OwnerID") - 1; COPY_STRING_MAXLEN(ownerId, S3_MAX_GRANTEE_USER_ID_SIZE); @@ -678,7 +701,7 @@ static int convert_simple_acl(char *aclXml, char *ownerId, grant->granteeType = S3GranteeTypeAllUsers; aclXml += (sizeof("All Users") - 1); } - else if (!strncmp(aclXml, "Log Delivery", + else if (!strncmp(aclXml, "Log Delivery", sizeof("Log Delivery") - 1)) { grant->granteeType = S3GranteeTypeLogDelivery; aclXml += (sizeof("Log Delivery") - 1); @@ -692,7 +715,7 @@ static int convert_simple_acl(char *aclXml, char *ownerId, } SKIP_SPACE(1); - + if (!strncmp(aclXml, "READ_ACP", sizeof("READ_ACP") - 1)) { grant->permission = S3PermissionReadACP; aclXml += (sizeof("READ_ACP") - 1); @@ -709,7 +732,7 @@ static int convert_simple_acl(char *aclXml, char *ownerId, grant->permission = S3PermissionWrite; aclXml += (sizeof("WRITE") - 1); } - else if (!strncmp(aclXml, "FULL_CONTROL", + else if (!strncmp(aclXml, "FULL_CONTROL", sizeof("FULL_CONTROL") - 1)) { grant->permission = S3PermissionFullControl; aclXml += (sizeof("FULL_CONTROL") - 1); @@ -741,7 +764,9 @@ static struct option longOptionsG[] = { "unencrypted", no_argument, 0, 'u' }, { "show-properties", no_argument, 0, 's' }, { "retries", required_argument, 0, 'r' }, + { "timeout", required_argument, 0, 't' }, { "verify-peer", no_argument, 0, 'v' }, + { "region", required_argument, 0, 'g' }, { 0, 0, 0, 0 } }; @@ -765,7 +790,7 @@ static S3Status responsePropertiesCallback printf("%s: %s\n", name, properties-> field); \ } \ } while (0) - + print_nonnull("Content-Type", contentType); print_nonnull("Request-Id", requestId); print_nonnull("Request-Id-2", requestId2); @@ -800,7 +825,7 @@ static S3Status responsePropertiesCallback // This callback does the same thing for every request type: saves the status // and error stuff in global variables static void responseCompleteCallback(S3Status status, - const S3ErrorDetails *error, + const S3ErrorDetails *error, void *callbackData) { (void) callbackData; @@ -827,8 +852,8 @@ static void responseCompleteCallback(S3Status status, "%s", " Extra Details:\n"); int i; for (i = 0; i < error->extraDetailsCount; i++) { - len += snprintf(&(errorDetailsG[len]), - sizeof(errorDetailsG) - len, " %s: %s\n", + len += snprintf(&(errorDetailsG[len]), + sizeof(errorDetailsG) - len, " %s: %s\n", error->extraDetails[i].name, error->extraDetails[i].value); } @@ -850,7 +875,7 @@ static void printListServiceHeader(int allDetails) printf("%-56s %-20s", " Bucket", " Created"); if (allDetails) { - printf(" %-64s %-12s", + printf(" %-64s %-12s", " Owner ID", "Display Name"); } @@ -865,7 +890,7 @@ static void printListServiceHeader(int allDetails) } -static S3Status listServiceCallback(const char *ownerId, +static S3Status listServiceCallback(const char *ownerId, const char *ownerDisplayName, const char *bucketName, int64_t creationDate, void *callbackData) @@ -888,7 +913,7 @@ static S3Status listServiceCallback(const char *ownerId, printf("%-56s %-20s", bucketName, timebuf); if (data->allDetails) { - printf(" %-64s %-12s", ownerId ? ownerId : "", + printf(" %-64s %-12s", ownerId ? ownerId : "", ownerDisplayName ? ownerDisplayName : ""); } printf("\n"); @@ -913,8 +938,8 @@ static void list_service(int allDetails) }; do { - S3_list_service(protocolG, accessKeyIdG, secretAccessKeyG, 0, 0, 0, - &listServiceHandler, &data); + S3_list_service(protocolG, accessKeyIdG, secretAccessKeyG, 0, 0, + awsRegionG, 0, timeoutMsG, &listServiceHandler, &data); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG == S3StatusOK) { @@ -957,8 +982,8 @@ static void test_bucket(int argc, char **argv, int optindex) char locationConstraint[64]; do { S3_test_bucket(protocolG, uriStyleG, accessKeyIdG, secretAccessKeyG, 0, - 0, bucketName, sizeof(locationConstraint), - locationConstraint, 0, &responseHandler, 0); + 0, bucketName, awsRegionG, sizeof(locationConstraint), + locationConstraint, 0, timeoutMsG, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); const char *result; @@ -1055,9 +1080,9 @@ static void create_bucket(int argc, char **argv, int optindex) }; do { - S3_create_bucket(protocolG, accessKeyIdG, secretAccessKeyG, 0, - 0, bucketName, cannedAcl, locationConstraint, 0, - &responseHandler, 0); + S3_create_bucket(protocolG, accessKeyIdG, secretAccessKeyG, 0, 0, + bucketName, awsRegionG, cannedAcl, locationConstraint, + 0, 0, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG == S3StatusOK) { @@ -1066,7 +1091,7 @@ static void create_bucket(int argc, char **argv, int optindex) else { printError(); } - + S3_deinitialize(); } @@ -1096,7 +1121,7 @@ static void delete_bucket(int argc, char **argv, int optindex) do { S3_delete_bucket(protocolG, uriStyleG, accessKeyIdG, secretAccessKeyG, - 0, 0, bucketName, 0, &responseHandler, 0); + 0, 0, bucketName, awsRegionG, 0, timeoutMsG, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG != S3StatusOK) { @@ -1120,12 +1145,12 @@ typedef struct list_bucket_callback_data static void printListBucketHeader(int allDetails) { - printf("%-50s %-20s %-5s", - " Key", + printf("%-50s %-20s %-5s", + " Key", " Last Modified", "Size"); if (allDetails) { - printf(" %-34s %-64s %-12s", - " ETag", + printf(" %-34s %-64s %-12s", + " ETag", " Owner ID", "Display Name"); } @@ -1142,13 +1167,13 @@ static void printListBucketHeader(int allDetails) static S3Status listBucketCallback(int isTruncated, const char *nextMarker, - int contentsCount, + int contentsCount, const S3ListBucketContent *contents, int commonPrefixesCount, const char **commonPrefixes, void *callbackData) { - list_bucket_callback_data *data = + list_bucket_callback_data *data = (list_bucket_callback_data *) callbackData; data->isTruncated = isTruncated; @@ -1160,13 +1185,13 @@ static S3Status listBucketCallback(int isTruncated, const char *nextMarker, nextMarker = contents[contentsCount - 1].key; } if (nextMarker) { - snprintf(data->nextMarker, sizeof(data->nextMarker), "%s", + snprintf(data->nextMarker, sizeof(data->nextMarker), "%s", nextMarker); } else { data->nextMarker[0] = 0; } - + if (contentsCount && !data->keyCount) { printListBucketHeader(data->allDetails); } @@ -1192,14 +1217,14 @@ static S3Status listBucketCallback(int isTruncated, const char *nextMarker, } else { time_t t = (time_t) content->lastModified; - strftime(timebuf, sizeof(timebuf), "%Y-%m-%dT%H:%M:%SZ", + strftime(timebuf, sizeof(timebuf), "%Y-%m-%dT%H:%M:%SZ", gmtime(&t)); char sizebuf[16]; if (content->size < 100000) { sprintf(sizebuf, "%5llu", (unsigned long long) content->size); } else if (content->size < (1024 * 1024)) { - sprintf(sizebuf, "%4lluK", + sprintf(sizebuf, "%4lluK", ((unsigned long long) content->size) / 1024ULL); } else if (content->size < (10 * 1024 * 1024)) { @@ -1208,8 +1233,8 @@ static S3Status listBucketCallback(int isTruncated, const char *nextMarker, sprintf(sizebuf, "%1.2fM", f); } else if (content->size < (1024 * 1024 * 1024)) { - sprintf(sizebuf, "%4lluM", - ((unsigned long long) content->size) / + sprintf(sizebuf, "%4lluM", + ((unsigned long long) content->size) / (1024ULL * 1024ULL)); } else { @@ -1220,9 +1245,9 @@ static S3Status listBucketCallback(int isTruncated, const char *nextMarker, printf("%-50s %s %s", content->key, timebuf, sizebuf); if (data->allDetails) { printf(" %-34s %-64s %-12s", - content->eTag, + content->eTag, content->ownerId ? content->ownerId : "", - content->ownerDisplayName ? + content->ownerDisplayName ? content->ownerDisplayName : ""); } printf("\n"); @@ -1244,7 +1269,7 @@ static void list_bucket(const char *bucketName, const char *prefix, int maxkeys, int allDetails) { S3_init(); - + S3BucketContext bucketContext = { 0, @@ -1253,7 +1278,8 @@ static void list_bucket(const char *bucketName, const char *prefix, uriStyleG, accessKeyIdG, secretAccessKeyG, - 0 + 0, + awsRegionG }; S3ListBucketHandler listBucketHandler = @@ -1276,7 +1302,7 @@ static void list_bucket(const char *bucketName, const char *prefix, data.isTruncated = 0; do { S3_list_bucket(&bucketContext, prefix, data.nextMarker, - delimiter, maxkeys, 0, &listBucketHandler, &data); + delimiter, maxkeys, 0, timeoutMsG, &listBucketHandler, &data); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG != S3StatusOK) { break; @@ -1309,6 +1335,7 @@ static void list(int argc, char **argv, int optindex) int maxkeys = 0, allDetails = 0; while (optindex < argc) { char *param = argv[optindex++]; + if (!strncmp(param, PREFIX_PREFIX, PREFIX_PREFIX_LEN)) { prefix = &(param[PREFIX_PREFIX_LEN]); } @@ -1324,7 +1351,7 @@ static void list(int argc, char **argv, int optindex) else if (!strncmp(param, ALL_DETAILS_PREFIX, ALL_DETAILS_PREFIX_LEN)) { const char *ad = &(param[ALL_DETAILS_PREFIX_LEN]); - if (!strcmp(ad, "true") || !strcmp(ad, "TRUE") || + if (!strcmp(ad, "true") || !strcmp(ad, "TRUE") || !strcmp(ad, "yes") || !strcmp(ad, "YES") || !strcmp(ad, "1")) { allDetails = 1; @@ -1340,7 +1367,7 @@ static void list(int argc, char **argv, int optindex) } if (bucketName) { - list_bucket(bucketName, prefix, marker, delimiter, maxkeys, + list_bucket(bucketName, prefix, marker, delimiter, maxkeys, allDetails); } else { @@ -1392,7 +1419,7 @@ typedef struct list_parts_callback_data typedef struct list_upload_callback_data { - char uploadId[1024]; + char uploadId[1024]; } abort_upload_callback_data; static void printListMultipartHeader(int allDetails) @@ -1404,28 +1431,28 @@ static void printListMultipartHeader(int allDetails) static void printListPartsHeader() { - printf("%-25s %-30s %-30s %-15s", - "LastModified", + printf("%-25s %-30s %-30s %-15s", + "LastModified", "PartNumber", "ETag", "SIZE"); - + printf("\n"); printf("--------------------- " - " ------------- " + " ------------- " "------------------------------- " - " -----"); + " -----"); printf("\n"); } static S3Status listMultipartCallback(int isTruncated, const char *nextKeyMarker, const char *nextUploadIdMarker, - int uploadsCount, + int uploadsCount, const S3ListMultipartUpload *uploads, int commonPrefixesCount, const char **commonPrefixes, void *callbackData) { - list_multiparts_callback_data *data = + list_multiparts_callback_data *data = (list_multiparts_callback_data *) callbackData; data->isTruncated = isTruncated; @@ -1438,7 +1465,7 @@ static S3Status listMultipartCallback(int isTruncated, const char *nextKeyMarker nextKeyMarker = uploads[uploadsCount - 1].key; }*/ if (nextKeyMarker) { - snprintf(data->nextKeyMarker, sizeof(data->nextKeyMarker), "%s", + snprintf(data->nextKeyMarker, sizeof(data->nextKeyMarker), "%s", nextKeyMarker); } else { @@ -1446,13 +1473,13 @@ static S3Status listMultipartCallback(int isTruncated, const char *nextKeyMarker } if (nextUploadIdMarker) { - snprintf(data->nextUploadIdMarker, sizeof(data->nextUploadIdMarker), "%s", + snprintf(data->nextUploadIdMarker, sizeof(data->nextUploadIdMarker), "%s", nextUploadIdMarker); } else { data->nextUploadIdMarker[0] = 0; } - + if (uploadsCount && !data->uploadCount) { printListMultipartHeader(data->allDetails); } @@ -1484,17 +1511,17 @@ static S3Status listMultipartCallback(int isTruncated, const char *nextKeyMarker } else { time_t t = (time_t) upload->initiated; - strftime(timebuf, sizeof(timebuf), "%Y-%m-%dT%H:%M:%SZ", - gmtime(&t)); + strftime(timebuf, sizeof(timebuf), "%Y-%m-%dT%H:%M:%SZ", + gmtime(&t)); printf("%-50s %s %-50s", upload->key, timebuf, upload->uploadId); if (data->allDetails) { printf(" %-34s %-64s %-12s %-64s %-12s", - upload->storageClass, + upload->storageClass, upload->ownerId ? upload->ownerId : "", - upload->ownerDisplayName ? + upload->ownerDisplayName ? upload->ownerDisplayName : "", upload->initiatorId ? upload->initiatorId : "", - upload->initiatorDisplayName ? + upload->initiatorDisplayName ? upload->initiatorDisplayName : ""); } printf("\n"); @@ -1512,18 +1539,18 @@ static S3Status listMultipartCallback(int isTruncated, const char *nextKeyMarker static S3Status listPartsCallback(int isTruncated, - const char *nextPartNumberMarker, - const char *initiatorId, - const char *initiatorDisplayName, - const char *ownerId, - const char *ownerDisplayName, - const char *storageClass, - int partsCount, - int handlePartsStart, - const S3ListPart *parts, - void *callbackData) + const char *nextPartNumberMarker, + const char *initiatorId, + const char *initiatorDisplayName, + const char *ownerId, + const char *ownerDisplayName, + const char *storageClass, + int partsCount, + int handlePartsStart, + const S3ListPart *parts, + void *callbackData) { - list_parts_callback_data *data = + list_parts_callback_data *data = (list_parts_callback_data *) callbackData; data->isTruncated = isTruncated; @@ -1539,7 +1566,7 @@ static S3Status listPartsCallback(int isTruncated, }*/ if (nextPartNumberMarker) { snprintf(data->nextPartNumberMarker, - sizeof(data->nextPartNumberMarker), "%s", + sizeof(data->nextPartNumberMarker), "%s", nextPartNumberMarker); } else { @@ -1547,7 +1574,7 @@ static S3Status listPartsCallback(int isTruncated, } if (initiatorId) { - snprintf(data->initiatorId, sizeof(data->initiatorId), "%s", + snprintf(data->initiatorId, sizeof(data->initiatorId), "%s", initiatorId); } else { @@ -1556,7 +1583,7 @@ static S3Status listPartsCallback(int isTruncated, if (initiatorDisplayName) { snprintf(data->initiatorDisplayName, - sizeof(data->initiatorDisplayName), "%s", + sizeof(data->initiatorDisplayName), "%s", initiatorDisplayName); } else { @@ -1564,7 +1591,7 @@ static S3Status listPartsCallback(int isTruncated, } if (ownerId) { - snprintf(data->ownerId, sizeof(data->ownerId), "%s", + snprintf(data->ownerId, sizeof(data->ownerId), "%s", ownerId); } else { @@ -1572,7 +1599,7 @@ static S3Status listPartsCallback(int isTruncated, } if (ownerDisplayName) { - snprintf(data->ownerDisplayName, sizeof(data->ownerDisplayName), "%s", + snprintf(data->ownerDisplayName, sizeof(data->ownerDisplayName), "%s", ownerDisplayName); } else { @@ -1580,13 +1607,13 @@ static S3Status listPartsCallback(int isTruncated, } if (storageClass) { - snprintf(data->storageClass, sizeof(data->storageClass), "%s", + snprintf(data->storageClass, sizeof(data->storageClass), "%s", storageClass); } else { data->storageClass[0] = 0; } - + if (partsCount && !data->partsCount && !data->noPrint) { printListPartsHeader(); } @@ -1610,7 +1637,7 @@ static S3Status listPartsCallback(int isTruncated, } } - data->partsCount += partsCount; + data->partsCount += partsCount; return S3StatusOK; } @@ -1628,7 +1655,7 @@ static void list_multipart_uploads(int argc, char **argv, int optindex) const char *encodingtype = 0, *uploadidmarker = 0; int maxuploads = 0, allDetails = 0; while (optindex < argc) { - char *param = argv[optindex++]; + char *param = argv[optindex++]; if (!strncmp(param, PREFIX_PREFIX, PREFIX_PREFIX_LEN)) { prefix = &(param[PREFIX_PREFIX_LEN]); } @@ -1652,7 +1679,7 @@ static void list_multipart_uploads(int argc, char **argv, int optindex) } else if (!strncmp(param, ALL_DETAILS_PREFIX, ALL_DETAILS_PREFIX_LEN)) { const char *ad = &(param[ALL_DETAILS_PREFIX_LEN]); - if (!strcmp(ad, "true") || !strcmp(ad, "TRUE") || + if (!strcmp(ad, "true") || !strcmp(ad, "TRUE") || !strcmp(ad, "yes") || !strcmp(ad, "YES") || !strcmp(ad, "1")) { allDetails = 1; @@ -1661,16 +1688,16 @@ static void list_multipart_uploads(int argc, char **argv, int optindex) else if (!bucketName) { bucketName = param; } - + else { fprintf(stderr, "\nERROR: Unknown param: %s\n", param); usageExit(stderr); } } if (bucketName) { - + S3_init(); - + S3BucketContext bucketContext = { 0, @@ -1679,7 +1706,8 @@ static void list_multipart_uploads(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, - 0 + 0, + awsRegionG }; S3ListMultipartUploadsHandler listMultipartUploadsHandler = @@ -1691,7 +1719,7 @@ static void list_multipart_uploads(int argc, char **argv, int optindex) list_multiparts_callback_data data; memset(&data, 0, sizeof(list_multiparts_callback_data)); - if (keymarker != 0) { + if (keymarker != 0) { snprintf(data.nextKeyMarker, sizeof(data.nextKeyMarker), "%s", keymarker); } @@ -1709,6 +1737,7 @@ static void list_multipart_uploads(int argc, char **argv, int optindex) data.nextKeyMarker, data.nextUploadIdMarker, encodingtype, delimiter, maxuploads, 0, + timeoutMsG, &listMultipartUploadsHandler, &data); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG != S3StatusOK) { @@ -1757,7 +1786,7 @@ static void list_parts(int argc, char **argv, int optindex) const char *encodingtype = 0; int allDetails = 0, maxparts = 0; while (optindex < argc) { - char *param = argv[optindex++]; + char *param = argv[optindex++]; if (!strncmp(param, UPLOAD_ID_PREFIX, UPLOAD_ID_PREFIX_LEN)) { uploadid = &(param[UPLOAD_ID_PREFIX_LEN]); } @@ -1778,21 +1807,21 @@ static void list_parts(int argc, char **argv, int optindex) else if (!strncmp(param, ALL_DETAILS_PREFIX, ALL_DETAILS_PREFIX_LEN)) { const char *ad = &(param[ALL_DETAILS_PREFIX_LEN]); - if (!strcmp(ad, "true") || !strcmp(ad, "TRUE") || + if (!strcmp(ad, "true") || !strcmp(ad, "TRUE") || !strcmp(ad, "yes") || !strcmp(ad, "YES") || !strcmp(ad, "1")) { allDetails = 1; } - } + } else { fprintf(stderr, "\nERROR: Unknown param: %s\n", param); usageExit(stderr); } } if (bucketName) { - + S3_init(); - + S3BucketContext bucketContext = { 0, @@ -1801,7 +1830,8 @@ static void list_parts(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, - 0 + 0, + awsRegionG }; S3ListPartsHandler listPartsHandler = @@ -1813,11 +1843,11 @@ static void list_parts(int argc, char **argv, int optindex) list_parts_callback_data data; memset(&data, 0, sizeof(list_parts_callback_data)); - if (partnumbermarker != 0) { + if (partnumbermarker != 0) { snprintf(data.nextPartNumberMarker, sizeof(data.nextPartNumberMarker), "%s", partnumbermarker); } - + data.partsCount = 0; data.allDetails = allDetails; data.noPrint = 0; @@ -1828,7 +1858,8 @@ static void list_parts(int argc, char **argv, int optindex) S3_list_parts(&bucketContext, key, data.nextPartNumberMarker, uploadid, encodingtype, maxparts, - 0, &listPartsHandler, &data); + 0, timeoutMsG, + &listPartsHandler, &data); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG != S3StatusOK) { break; @@ -1846,9 +1877,7 @@ static void list_parts(int argc, char **argv, int optindex) } S3_deinitialize(); - } - } @@ -1859,7 +1888,7 @@ static void abort_multipart_upload(int argc, char **argv, int optindex) "\n"); return; } - + // Split bucket/key char *slash = argv[optindex]; while (*slash && (*slash != '/')) { @@ -1876,22 +1905,22 @@ static void abort_multipart_upload(int argc, char **argv, int optindex) const char *key = slash; const char *uploadid = 0; while (optindex < argc) { - char *param = argv[optindex++]; + char *param = argv[optindex++]; if (!strncmp(param, UPLOAD_ID_PREFIX, UPLOAD_ID_PREFIX_LEN)) { uploadid = &(param[UPLOAD_ID_PREFIX_LEN]); } else if (!strncmp(param, FILENAME_PREFIX, FILENAME_PREFIX_LEN)) { key = &(param[FILENAME_PREFIX_LEN]); - } + } else { fprintf(stderr, "\nERROR: Unknown param: %s\n", param); usageExit(stderr); } } if (bucketName) { - + S3_init(); - + S3BucketContext bucketContext = { 0, @@ -1900,7 +1929,8 @@ static void abort_multipart_upload(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, - 0 + 0, + awsRegionG }; S3AbortMultipartUploadHandler abortMultipartUploadHandler = @@ -1912,7 +1942,7 @@ static void abort_multipart_upload(int argc, char **argv, int optindex) list_multiparts_callback_data data; memset(&data, 0, sizeof(list_multiparts_callback_data)); - if (keymarker != 0) { + if (keymarker != 0) { snprintf(data.nextKeyMarker, sizeof(data.nextKeyMarker), "%s", keymarker); } @@ -1920,16 +1950,16 @@ static void abort_multipart_upload(int argc, char **argv, int optindex) snprintf(data.nextUploadIdMarker, sizeof(data.nextUploadIdMarker), "%s", uploadidmarker); } - + data.uploadCount = 0; data.allDetails = allDetails; */ do { S3_abort_multipart_upload(&bucketContext, key, uploadid, - &abortMultipartUploadHandler); + timeoutMsG, &abortMultipartUploadHandler); } while (S3_status_is_retryable(statusG) && should_retry()); - + S3_deinitialize(); } } @@ -1954,7 +1984,7 @@ static void delete_object(int argc, char **argv, int optindex) const char *key = slash; S3_init(); - + S3BucketContext bucketContext = { 0, @@ -1963,17 +1993,18 @@ static void delete_object(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, - 0 + 0, + awsRegionG }; S3ResponseHandler responseHandler = - { + { 0, &responseCompleteCallback }; do { - S3_delete_object(&bucketContext, key, 0, &responseHandler, 0); + S3_delete_object(&bucketContext, key, 0, timeoutMsG, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); if ((statusG != S3StatusOK) && @@ -2000,9 +2031,9 @@ typedef struct put_object_callback_data static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) { - put_object_callback_data *data = + put_object_callback_data *data = (put_object_callback_data *) callbackData; - + int ret = 0; if (data->contentLength) { @@ -2022,10 +2053,10 @@ static int putObjectDataCallback(int bufferSize, char *buffer, if (data->contentLength && !data->noStatus) { // Avoid a weird bug in MingW, which won't print the second integer // value properly when it's in the same call, so print separately - printf("%llu bytes remaining ", + printf("%llu bytes remaining ", (unsigned long long) data->totalContentLength); printf("(%d%% complete) ...\n", - (int) (((data->totalOriginalContentLength - + (int) (((data->totalOriginalContentLength - data->totalContentLength) * 100) / data->totalOriginalContentLength)); } @@ -2033,7 +2064,7 @@ static int putObjectDataCallback(int bufferSize, char *buffer, return ret; } -#define MULTIPART_CHUNK_SIZE (15 << 20) // multipart is 15M +#define MULTIPART_CHUNK_SIZE (768 << 20) // multipart is 768M typedef struct MultipartPartData { put_object_callback_data put_object_data; @@ -2063,6 +2094,7 @@ S3Status MultipartResponseProperiesCallback return S3StatusOK; } + static int multipartPutXmlCallback(int bufferSize, char *buffer, void *callbackData) { @@ -2078,7 +2110,7 @@ static int multipartPutXmlCallback(int bufferSize, char *buffer, } -static int try_get_parts_info(const char *bucketName, const char *key, +static int try_get_parts_info(const char *bucketName, const char *key, UploadManager *manager) { S3BucketContext bucketContext = @@ -2089,7 +2121,8 @@ static int try_get_parts_info(const char *bucketName, const char *key, uriStyleG, accessKeyIdG, secretAccessKeyG, - 0 + 0, + awsRegionG }; S3ListPartsHandler listPartsHandler = @@ -2101,7 +2134,7 @@ static int try_get_parts_info(const char *bucketName, const char *key, list_parts_callback_data data; memset(&data, 0, sizeof(list_parts_callback_data)); - + data.partsCount = 0; data.allDetails = 0; data.manager = manager; @@ -2110,7 +2143,7 @@ static int try_get_parts_info(const char *bucketName, const char *key, data.isTruncated = 0; do { S3_list_parts(&bucketContext, key, data.nextPartNumberMarker, - manager->upload_id, 0, 0, 0, &listPartsHandler, + manager->upload_id, 0, 0, 0, timeoutMsG, &listPartsHandler, &data); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG != S3StatusOK) { @@ -2127,10 +2160,11 @@ static int try_get_parts_info(const char *bucketName, const char *key, printError(); return -1; } - + return 0; } + static void put_object(int argc, char **argv, int optindex, const char *srcBucketName, const char *srcKey, unsigned long long srcSize) { @@ -2170,37 +2204,37 @@ static void put_object(int argc, char **argv, int optindex, if (!strncmp(param, FILENAME_PREFIX, FILENAME_PREFIX_LEN)) { filename = &(param[FILENAME_PREFIX_LEN]); } - else if (!strncmp(param, CONTENT_LENGTH_PREFIX, + else if (!strncmp(param, CONTENT_LENGTH_PREFIX, CONTENT_LENGTH_PREFIX_LEN)) { contentLength = convertInt(&(param[CONTENT_LENGTH_PREFIX_LEN]), "contentLength"); - if (contentLength > (5LL * 1024 * 1024 * 1024)) { + if (contentLength > (5LL * 1024 * 1024 * 1024 * 1024)) { fprintf(stderr, "\nERROR: contentLength must be no greater " - "than 5 GB\n"); + "than 5 TB\n"); usageExit(stderr); } } - else if (!strncmp(param, CACHE_CONTROL_PREFIX, + else if (!strncmp(param, CACHE_CONTROL_PREFIX, CACHE_CONTROL_PREFIX_LEN)) { cacheControl = &(param[CACHE_CONTROL_PREFIX_LEN]); } - else if (!strncmp(param, CONTENT_TYPE_PREFIX, + else if (!strncmp(param, CONTENT_TYPE_PREFIX, CONTENT_TYPE_PREFIX_LEN)) { contentType = &(param[CONTENT_TYPE_PREFIX_LEN]); } else if (!strncmp(param, MD5_PREFIX, MD5_PREFIX_LEN)) { md5 = &(param[MD5_PREFIX_LEN]); } - else if (!strncmp(param, CONTENT_DISPOSITION_FILENAME_PREFIX, + else if (!strncmp(param, CONTENT_DISPOSITION_FILENAME_PREFIX, CONTENT_DISPOSITION_FILENAME_PREFIX_LEN)) { - contentDispositionFilename = + contentDispositionFilename = &(param[CONTENT_DISPOSITION_FILENAME_PREFIX_LEN]); } - else if (!strncmp(param, CONTENT_ENCODING_PREFIX, + else if (!strncmp(param, CONTENT_ENCODING_PREFIX, CONTENT_ENCODING_PREFIX_LEN)) { contentEncoding = &(param[CONTENT_ENCODING_PREFIX_LEN]); } - else if (!strncmp(param, UPLOAD_ID_PREFIX, + else if (!strncmp(param, UPLOAD_ID_PREFIX, UPLOAD_ID_PREFIX_LEN)) { uploadId = &(param[UPLOAD_ID_PREFIX_LEN]); } @@ -2215,7 +2249,7 @@ static void put_object(int argc, char **argv, int optindex, else if (!strncmp(param, X_AMZ_META_PREFIX, X_AMZ_META_PREFIX_LEN)) { if (metaPropertiesCount == S3_MAX_METADATA_COUNT) { fprintf(stderr, "\nERROR: Too many x-amz-meta- properties, " - "limit %lu: %s\n", + "limit %lu: %s\n", (unsigned long) S3_MAX_METADATA_COUNT, param); usageExit(stderr); } @@ -2235,7 +2269,7 @@ static void put_object(int argc, char **argv, int optindex, else if (!strncmp(param, USE_SERVER_SIDE_ENCRYPTION_PREFIX, USE_SERVER_SIDE_ENCRYPTION_PREFIX_LEN)) { const char *val = &(param[USE_SERVER_SIDE_ENCRYPTION_PREFIX_LEN]); - if (!strcmp(val, "true") || !strcmp(val, "TRUE") || + if (!strcmp(val, "true") || !strcmp(val, "TRUE") || !strcmp(val, "yes") || !strcmp(val, "YES") || !strcmp(val, "1")) { useServerSideEncryption = 1; @@ -2255,6 +2289,9 @@ static void put_object(int argc, char **argv, int optindex, else if (!strcmp(val, "public-read-write")) { cannedAcl = S3CannedAclPublicReadWrite; } + else if (!strcmp(val, "bucket-owner-full-control")) { + cannedAcl = S3CannedAclBucketOwnerFullControl; + } else if (!strcmp(val, "authenticated-read")) { cannedAcl = S3CannedAclAuthenticatedRead; } @@ -2265,7 +2302,7 @@ static void put_object(int argc, char **argv, int optindex, } else if (!strncmp(param, NO_STATUS_PREFIX, NO_STATUS_PREFIX_LEN)) { const char *ns = &(param[NO_STATUS_PREFIX_LEN]); - if (!strcmp(ns, "true") || !strcmp(ns, "TRUE") || + if (!strcmp(ns, "true") || !strcmp(ns, "TRUE") || !strcmp(ns, "yes") || !strcmp(ns, "YES") || !strcmp(ns, "1")) { noStatus = 1; @@ -2336,13 +2373,13 @@ static void put_object(int argc, char **argv, int optindex, } data.totalContentLength = - data.totalOriginalContentLength = + data.totalOriginalContentLength = data.contentLength = data.originalContentLength = contentLength; S3_init(); - + S3BucketContext bucketContext = { 0, @@ -2351,7 +2388,8 @@ static void put_object(int argc, char **argv, int optindex, uriStyleG, accessKeyIdG, secretAccessKeyG, - 0 + 0, + awsRegionG }; S3PutProperties putProperties = @@ -2367,7 +2405,7 @@ static void put_object(int argc, char **argv, int optindex, metaProperties, useServerSideEncryption }; - + if (contentLength <= MULTIPART_CHUNK_SIZE) { S3PutObjectHandler putObjectHandler = { @@ -2377,7 +2415,7 @@ static void put_object(int argc, char **argv, int optindex, do { S3_put_object(&bucketContext, key, contentLength, &putProperties, 0, - &putObjectHandler, &data); + 0, &putObjectHandler, &data); } while (S3_status_is_retryable(statusG) && should_retry()); if (data.infile) { @@ -2408,6 +2446,7 @@ static void put_object(int argc, char **argv, int optindex, MULTIPART_CHUNK_SIZE); MultipartPartData partData; + memset(&partData, 0, sizeof(MultipartPartData)); int partContentLength = 0; S3MultipartInitialHandler handler = { @@ -2415,7 +2454,7 @@ static void put_object(int argc, char **argv, int optindex, &responsePropertiesCallback, &responseCompleteCallback }, - &initial_multipart_callback + &initial_multipart_callback }; S3PutObjectHandler putObjectHandler = { @@ -2425,29 +2464,29 @@ static void put_object(int argc, char **argv, int optindex, S3MultipartCommitHandler commit_handler = { { - &responsePropertiesCallback,&responseCompleteCallback + &responsePropertiesCallback,&responseCompleteCallback }, &multipartPutXmlCallback, 0 }; - + manager.etags = (char **) malloc(sizeof(char *) * totalSeq); manager.next_etags_pos = 0; - + if (uploadId) { manager.upload_id = strdup(uploadId); manager.remaining = contentLength; - if(!try_get_parts_info(bucketName, key, &manager)) { + if (!try_get_parts_info(bucketName, key, &manager)) { fseek(data.infile, -(manager.remaining), 2); contentLength = manager.remaining; goto upload; - }else { + } else { goto clean; } } - + do { - S3_initiate_multipart(&bucketContext,key,0, &handler,0, &manager); + S3_initiate_multipart(&bucketContext, key,0, &handler,0, timeoutMsG, &manager); } while (S3_status_is_retryable(statusG) && should_retry()); if (manager.upload_id == 0 || statusG != S3StatusOK) { @@ -2455,13 +2494,14 @@ static void put_object(int argc, char **argv, int optindex, goto clean; } -upload: +upload: todoContentLength -= MULTIPART_CHUNK_SIZE * manager.next_etags_pos; for (seq = manager.next_etags_pos + 1; seq <= totalSeq; seq++) { - memset(&partData, 0, sizeof(MultipartPartData)); partData.manager = &manager; partData.seq = seq; - partData.put_object_data = data; + if (partData.put_object_data.gb==NULL) { + partData.put_object_data = data; + } partContentLength = ((contentLength > MULTIPART_CHUNK_SIZE) ? MULTIPART_CHUNK_SIZE : contentLength); printf("%s Part Seq %d, length=%d\n", srcSize ? "Copying" : "Sending", seq, partContentLength); @@ -2480,7 +2520,8 @@ static void put_object(int argc, char **argv, int optindex, uriStyleG, accessKeyIdG, secretAccessKeyG, - 0 + 0, + awsRegionG }; S3ResponseHandler copyResponseHandler = { &responsePropertiesCallback, &responseCompleteCallback }; @@ -2490,16 +2531,21 @@ static void put_object(int argc, char **argv, int optindex, unsigned long long count = partContentLength - 1; // Inclusive for copies // The default copy callback tries to set this for us, need to allocate here manager.etags[seq-1] = malloc(512); // TBD - magic #! Isa there a max etag defined? - S3_copy_object_range(&srcBucketContext, srcKey, bucketName, key, - seq, manager.upload_id, - startOffset, count, - &putProperties, - &lastModified, 512 /*TBD - magic # */, manager.etags[seq-1], 0, - ©ResponseHandler, 0); + S3_copy_object_range(&srcBucketContext, srcKey, + bucketName, key, + seq, manager.upload_id, + startOffset, count, + &putProperties, + &lastModified, 512 /*TBD - magic # */, + manager.etags[seq-1], 0, + timeoutMsG, + ©ResponseHandler, 0); } else { S3_upload_part(&bucketContext, key, &putProperties, &putObjectHandler, seq, manager.upload_id, - partContentLength,0, &partData); + partContentLength, + 0, timeoutMsG, + &partData); } } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG != S3StatusOK) { @@ -2509,7 +2555,7 @@ static void put_object(int argc, char **argv, int optindex, contentLength -= MULTIPART_CHUNK_SIZE; todoContentLength -= MULTIPART_CHUNK_SIZE; } - + int i; int size = 0; size += growbuffer_append(&(manager.gb), "", @@ -2528,7 +2574,7 @@ static void put_object(int argc, char **argv, int optindex, do { S3_complete_multipart_upload(&bucketContext, key, &commit_handler, manager.upload_id, manager.remaining, - 0, &manager); + 0, timeoutMsG, &manager); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG != S3StatusOK) { printError(); @@ -2545,7 +2591,7 @@ static void put_object(int argc, char **argv, int optindex, growbuffer_destroy(manager.gb); free(manager.etags); } - + S3_deinitialize(); } @@ -2614,7 +2660,8 @@ static void copy_object(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, - 0 + 0, + awsRegionG }; S3ListBucketHandler listBucketHandler = { @@ -2624,10 +2671,13 @@ static void copy_object(int argc, char **argv, int optindex) // Find size of existing key to determine if MP required do { S3_list_bucket(&listBucketContext, sourceKey, NULL, - ".", 1, 0, &listBucketHandler, &sourceSize); + ".", 1, 0, + timeoutMsG, &listBucketHandler, &sourceSize); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG != S3StatusOK) { - fprintf(stderr, "\nERROR: Unable to get source object size\n"); + fprintf(stderr, "\nERROR: Unable to get source object size (%s)\n", + S3_get_status_name(statusG)); + fprintf(stderr, "%s\n", errorDetailsG); exit(1); } if (sourceSize > MULTIPART_CHUNK_SIZE) { @@ -2662,23 +2712,23 @@ static void copy_object(int argc, char **argv, int optindex) while (optindex < argc) { char *param = argv[optindex++]; - if (!strncmp(param, CACHE_CONTROL_PREFIX, + if (!strncmp(param, CACHE_CONTROL_PREFIX, CACHE_CONTROL_PREFIX_LEN)) { cacheControl = &(param[CACHE_CONTROL_PREFIX_LEN]); anyPropertiesSet = 1; } - else if (!strncmp(param, CONTENT_TYPE_PREFIX, + else if (!strncmp(param, CONTENT_TYPE_PREFIX, CONTENT_TYPE_PREFIX_LEN)) { contentType = &(param[CONTENT_TYPE_PREFIX_LEN]); anyPropertiesSet = 1; } - else if (!strncmp(param, CONTENT_DISPOSITION_FILENAME_PREFIX, + else if (!strncmp(param, CONTENT_DISPOSITION_FILENAME_PREFIX, CONTENT_DISPOSITION_FILENAME_PREFIX_LEN)) { - contentDispositionFilename = + contentDispositionFilename = &(param[CONTENT_DISPOSITION_FILENAME_PREFIX_LEN]); anyPropertiesSet = 1; } - else if (!strncmp(param, CONTENT_ENCODING_PREFIX, + else if (!strncmp(param, CONTENT_ENCODING_PREFIX, CONTENT_ENCODING_PREFIX_LEN)) { contentEncoding = &(param[CONTENT_ENCODING_PREFIX_LEN]); anyPropertiesSet = 1; @@ -2695,7 +2745,7 @@ static void copy_object(int argc, char **argv, int optindex) else if (!strncmp(param, X_AMZ_META_PREFIX, X_AMZ_META_PREFIX_LEN)) { if (metaPropertiesCount == S3_MAX_METADATA_COUNT) { fprintf(stderr, "\nERROR: Too many x-amz-meta- properties, " - "limit %lu: %s\n", + "limit %lu: %s\n", (unsigned long) S3_MAX_METADATA_COUNT, param); usageExit(stderr); } @@ -2715,7 +2765,7 @@ static void copy_object(int argc, char **argv, int optindex) } else if (!strncmp(param, USE_SERVER_SIDE_ENCRYPTION_PREFIX, USE_SERVER_SIDE_ENCRYPTION_PREFIX_LEN)) { - if (!strcmp(param, "true") || !strcmp(param, "TRUE") || + if (!strcmp(param, "true") || !strcmp(param, "TRUE") || !strcmp(param, "yes") || !strcmp(param, "YES") || !strcmp(param, "1")) { useServerSideEncryption = 1; @@ -2751,7 +2801,6 @@ static void copy_object(int argc, char **argv, int optindex) } } - S3BucketContext bucketContext = { 0, @@ -2760,7 +2809,8 @@ static void copy_object(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, - 0 + 0, + awsRegionG }; S3PutProperties putProperties = @@ -2778,7 +2828,7 @@ static void copy_object(int argc, char **argv, int optindex) }; S3ResponseHandler responseHandler = - { + { &responsePropertiesCallback, &responseCompleteCallback }; @@ -2790,6 +2840,7 @@ static void copy_object(int argc, char **argv, int optindex) S3_copy_object(&bucketContext, sourceKey, destinationBucketName, destinationKey, anyPropertiesSet ? &putProperties : 0, &lastModified, sizeof(eTag), eTag, 0, + timeoutMsG, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); @@ -2821,8 +2872,8 @@ static S3Status getObjectDataCallback(int bufferSize, const char *buffer, FILE *outfile = (FILE *) callbackData; size_t wrote = fwrite(buffer, 1, bufferSize, outfile); - - return ((wrote < (size_t) bufferSize) ? + + return ((wrote < (size_t) bufferSize) ? S3StatusAbortedByCallback : S3StatusOK); } @@ -2859,7 +2910,7 @@ static void get_object(int argc, char **argv, int optindex) if (!strncmp(param, FILENAME_PREFIX, FILENAME_PREFIX_LEN)) { filename = &(param[FILENAME_PREFIX_LEN]); } - else if (!strncmp(param, IF_MODIFIED_SINCE_PREFIX, + else if (!strncmp(param, IF_MODIFIED_SINCE_PREFIX, IF_MODIFIED_SINCE_PREFIX_LEN)) { // Parse ifModifiedSince ifModifiedSince = parseIso8601Time @@ -2870,7 +2921,7 @@ static void get_object(int argc, char **argv, int optindex) usageExit(stderr); } } - else if (!strncmp(param, IF_NOT_MODIFIED_SINCE_PREFIX, + else if (!strncmp(param, IF_NOT_MODIFIED_SINCE_PREFIX, IF_NOT_MODIFIED_SINCE_PREFIX_LEN)) { // Parse ifModifiedSince ifNotModifiedSince = parseIso8601Time @@ -2916,7 +2967,7 @@ static void get_object(int argc, char **argv, int optindex) // unmodified outfile = fopen(filename, "r+" FOPEN_EXTRA_FLAGS); } - + if (!outfile) { fprintf(stderr, "\nERROR: Failed to open output file %s: ", filename); @@ -2933,7 +2984,7 @@ static void get_object(int argc, char **argv, int optindex) } S3_init(); - + S3BucketContext bucketContext = { 0, @@ -2942,7 +2993,8 @@ static void get_object(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, - 0 + 0, + awsRegionG }; S3GetConditions getConditions = @@ -2961,7 +3013,7 @@ static void get_object(int argc, char **argv, int optindex) do { S3_get_object(&bucketContext, key, &getConditions, startByte, - byteCount, 0, &getObjectHandler, outfile); + byteCount, 0, 0, &getObjectHandler, outfile); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG != S3StatusOK) { @@ -2982,7 +3034,7 @@ static void head_object(int argc, char **argv, int optindex) fprintf(stderr, "\nERROR: Missing parameter: bucket/key\n"); usageExit(stderr); } - + // Head implies showing response properties showResponsePropertiesG = 1; @@ -3008,7 +3060,7 @@ static void head_object(int argc, char **argv, int optindex) } S3_init(); - + S3BucketContext bucketContext = { 0, @@ -3017,17 +3069,18 @@ static void head_object(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, - 0 + 0, + awsRegionG }; S3ResponseHandler responseHandler = - { + { &responsePropertiesCallback, &responseCompleteCallback }; do { - S3_head_object(&bucketContext, key, 0, &responseHandler, 0); + S3_head_object(&bucketContext, key, 0, 0, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); if ((statusG != S3StatusOK) && @@ -3064,9 +3117,10 @@ static void generate_query_string(int argc, char **argv, int optindex) key = 0; } - int64_t expires = -1; + int expires = -1; const char *resource = 0; + const char *httpMethod = "GET"; while (optindex < argc) { char *param = argv[optindex++]; @@ -3081,6 +3135,9 @@ static void generate_query_string(int argc, char **argv, int optindex) else if (!strncmp(param, RESOURCE_PREFIX, RESOURCE_PREFIX_LEN)) { resource = &(param[RESOURCE_PREFIX_LEN]); } + else if (!strncmp(param, HTTP_METHOD_PREFIX, HTTP_METHOD_PREFIX_LEN)) { + httpMethod = &(param[HTTP_METHOD_PREFIX_LEN]); + } else { fprintf(stderr, "\nERROR: Unknown param: %s\n", param); usageExit(stderr); @@ -3088,7 +3145,7 @@ static void generate_query_string(int argc, char **argv, int optindex) } S3_init(); - + S3BucketContext bucketContext = { 0, @@ -3097,14 +3154,15 @@ static void generate_query_string(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, - 0 + 0, + awsRegionG }; char buffer[S3_MAX_AUTHENTICATED_QUERY_STRING_SIZE]; S3Status status = S3_generate_authenticated_query_string - (buffer, &bucketContext, key, expires, resource); - + (buffer, &bucketContext, key, expires, resource, httpMethod); + if (status != S3StatusOK) { printf("Failed to generate authenticated query string: %s\n", S3_get_status_name(status)); @@ -3169,7 +3227,7 @@ void get_acl(int argc, char **argv, int optindex) // unmodified outfile = fopen(filename, "r+" FOPEN_EXTRA_FLAGS); } - + if (!outfile) { fprintf(stderr, "\nERROR: Failed to open output file %s: ", filename); @@ -3200,7 +3258,8 @@ void get_acl(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, - 0 + 0, + awsRegionG }; S3ResponseHandler responseHandler = @@ -3210,13 +3269,14 @@ void get_acl(int argc, char **argv, int optindex) }; do { - S3_get_acl(&bucketContext, key, ownerId, ownerDisplayName, - &aclGrantCount, aclGrants, 0, &responseHandler, 0); + S3_get_acl(&bucketContext, key, ownerId, ownerDisplayName, + &aclGrantCount, aclGrants, 0, + timeoutMsG, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG == S3StatusOK) { fprintf(outfile, "OwnerID %s %s\n", ownerId, ownerDisplayName); - fprintf(outfile, "%-6s %-90s %-12s\n", " Type", + fprintf(outfile, "%-6s %-90s %-12s\n", " Type", " User Identifier", " Permission"); fprintf(outfile, "------ " @@ -3226,7 +3286,7 @@ void get_acl(int argc, char **argv, int optindex) for (i = 0; i < aclGrantCount; i++) { S3AclGrant *grant = &(aclGrants[i]); const char *type; - char composedId[S3_MAX_GRANTEE_USER_ID_SIZE + + char composedId[S3_MAX_GRANTEE_USER_ID_SIZE + S3_MAX_GRANTEE_DISPLAY_NAME_SIZE + 16]; const char *id; @@ -3340,10 +3400,10 @@ void set_acl(int argc, char **argv, int optindex) // Read in the complete ACL char aclBuf[65536]; - aclBuf[fread(aclBuf, 1, sizeof(aclBuf), infile)] = 0; + aclBuf[fread(aclBuf, 1, sizeof(aclBuf) - 1, infile)] = 0; char ownerId[S3_MAX_GRANTEE_USER_ID_SIZE]; char ownerDisplayName[S3_MAX_GRANTEE_DISPLAY_NAME_SIZE]; - + // Parse it int aclGrantCount; S3AclGrant aclGrants[S3_MAX_ACL_GRANT_COUNT]; @@ -3364,7 +3424,8 @@ void set_acl(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, - 0 + 0, + awsRegionG }; S3ResponseHandler responseHandler = @@ -3375,9 +3436,182 @@ void set_acl(int argc, char **argv, int optindex) do { S3_set_acl(&bucketContext, key, ownerId, ownerDisplayName, - aclGrantCount, aclGrants, 0, &responseHandler, 0); + aclGrantCount, aclGrants, 0, + timeoutMsG, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); - + + if (statusG != S3StatusOK) { + printError(); + } + + fclose(infile); + + S3_deinitialize(); +} + +// get lifecycle ------------------------------------------------------------------- + +void get_lifecycle(int argc, char **argv, int optindex) +{ + if (optindex == argc) { + fprintf(stderr, "\nERROR: Missing parameter: bucket\n"); + usageExit(stderr); + } + + const char *bucketName = argv[optindex++]; + + const char *filename = 0; + + while (optindex < argc) { + char *param = argv[optindex++]; + if (!strncmp(param, FILENAME_PREFIX, FILENAME_PREFIX_LEN)) { + filename = &(param[FILENAME_PREFIX_LEN]); + } + else { + fprintf(stderr, "\nERROR: Unknown param: %s\n", param); + usageExit(stderr); + } + } + + FILE *outfile = 0; + + if (filename) { + // Stat the file, and if it doesn't exist, open it in w mode + struct stat buf; + if (stat(filename, &buf) == -1) { + outfile = fopen(filename, "w" FOPEN_EXTRA_FLAGS); + } + else { + // Open in r+ so that we don't truncate the file, just in case + // there is an error and we write no bytes, we leave the file + // unmodified + outfile = fopen(filename, "r+" FOPEN_EXTRA_FLAGS); + } + + if (!outfile) { + fprintf(stderr, "\nERROR: Failed to open output file %s: ", + filename); + perror(0); + exit(-1); + } + } + else if (showResponsePropertiesG) { + fprintf(stderr, "\nERROR: getlifecycle -s requires a filename parameter\n"); + usageExit(stderr); + } + else { + outfile = stdout; + } + + char lifecycleBuffer[64 * 1024]; + + S3_init(); + + S3BucketContext bucketContext = + { + 0, + bucketName, + protocolG, + uriStyleG, + accessKeyIdG, + secretAccessKeyG, + 0, + awsRegionG + }; + + S3ResponseHandler responseHandler = + { + &responsePropertiesCallback, + &responseCompleteCallback + }; + + do { + S3_get_lifecycle(&bucketContext, + lifecycleBuffer, sizeof(lifecycleBuffer), + 0, timeoutMsG, &responseHandler, 0); + } while (S3_status_is_retryable(statusG) && should_retry()); + + if (statusG == S3StatusOK) { + fprintf(outfile, "%s", lifecycleBuffer); + } + else { + printError(); + } + + fclose(outfile); + + S3_deinitialize(); +} + + +// set lifecycle ------------------------------------------------------------------- + +void set_lifecycle(int argc, char **argv, int optindex) +{ + if (optindex == argc) { + fprintf(stderr, "\nERROR: Missing parameter: bucket\n"); + usageExit(stderr); + } + + const char *bucketName = argv[optindex++]; + + const char *filename = 0; + + while (optindex < argc) { + char *param = argv[optindex++]; + if (!strncmp(param, FILENAME_PREFIX, FILENAME_PREFIX_LEN)) { + filename = &(param[FILENAME_PREFIX_LEN]); + } + else { + fprintf(stderr, "\nERROR: Unknown param: %s\n", param); + usageExit(stderr); + } + } + + FILE *infile; + + if (filename) { + if (!(infile = fopen(filename, "r" FOPEN_EXTRA_FLAGS))) { + fprintf(stderr, "\nERROR: Failed to open input file %s: ", + filename); + perror(0); + exit(-1); + } + } + else { + infile = stdin; + } + + // Read in the complete ACL + char lifecycleBuf[65536]; + lifecycleBuf[fread(lifecycleBuf, 1, sizeof(lifecycleBuf) - 1, infile)] = 0; + + S3_init(); + + S3BucketContext bucketContext = + { + 0, + bucketName, + protocolG, + uriStyleG, + accessKeyIdG, + secretAccessKeyG, + 0, + awsRegionG + }; + + S3ResponseHandler responseHandler = + { + &responsePropertiesCallback, + &responseCompleteCallback + }; + + do { + S3_set_lifecycle(&bucketContext, + lifecycleBuf, + 0, timeoutMsG, &responseHandler, 0); + } while (S3_status_is_retryable(statusG) && should_retry()); + if (statusG != S3StatusOK) { printError(); } @@ -3425,7 +3659,7 @@ void get_logging(int argc, char **argv, int optindex) // unmodified outfile = fopen(filename, "r+" FOPEN_EXTRA_FLAGS); } - + if (!outfile) { fprintf(stderr, "\nERROR: Failed to open output file %s: ", filename); @@ -3457,7 +3691,8 @@ void get_logging(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, - 0 + 0, + awsRegionG }; S3ResponseHandler responseHandler = @@ -3468,7 +3703,8 @@ void get_logging(int argc, char **argv, int optindex) do { S3_get_server_access_logging(&bucketContext, targetBucket, targetPrefix, - &aclGrantCount, aclGrants, 0, + &aclGrantCount, aclGrants, 0, + timeoutMsG, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); @@ -3478,7 +3714,7 @@ void get_logging(int argc, char **argv, int optindex) if (targetPrefix[0]) { printf("Target Prefix: %s\n", targetPrefix); } - fprintf(outfile, "%-6s %-90s %-12s\n", " Type", + fprintf(outfile, "%-6s %-90s %-12s\n", " Type", " User Identifier", " Permission"); fprintf(outfile, "------ " @@ -3488,10 +3724,10 @@ void get_logging(int argc, char **argv, int optindex) for (i = 0; i < aclGrantCount; i++) { S3AclGrant *grant = &(aclGrants[i]); const char *type; - char composedId[S3_MAX_GRANTEE_USER_ID_SIZE + + char composedId[S3_MAX_GRANTEE_USER_ID_SIZE + S3_MAX_GRANTEE_DISPLAY_NAME_SIZE + 16]; const char *id; - + switch (grant->granteeType) { case S3GranteeTypeAmazonCustomerByEmail: type = "Email"; @@ -3566,7 +3802,7 @@ void set_logging(int argc, char **argv, int optindex) if (!strncmp(param, TARGET_BUCKET_PREFIX, TARGET_BUCKET_PREFIX_LEN)) { targetBucket = &(param[TARGET_BUCKET_PREFIX_LEN]); } - else if (!strncmp(param, TARGET_PREFIX_PREFIX, + else if (!strncmp(param, TARGET_PREFIX_PREFIX, TARGET_PREFIX_PREFIX_LEN)) { targetPrefix = &(param[TARGET_PREFIX_PREFIX_LEN]); } @@ -3584,7 +3820,7 @@ void set_logging(int argc, char **argv, int optindex) if (targetBucket) { FILE *infile; - + if (filename) { if (!(infile = fopen(filename, "r" FOPEN_EXTRA_FLAGS))) { fprintf(stderr, "\nERROR: Failed to open input file %s: ", @@ -3602,7 +3838,7 @@ void set_logging(int argc, char **argv, int optindex) aclBuf[fread(aclBuf, 1, sizeof(aclBuf), infile)] = 0; char ownerId[S3_MAX_GRANTEE_USER_ID_SIZE]; char ownerDisplayName[S3_MAX_GRANTEE_DISPLAY_NAME_SIZE]; - + // Parse it if (!convert_simple_acl(aclBuf, ownerId, ownerDisplayName, &aclGrantCount, aclGrants)) { @@ -3624,7 +3860,8 @@ void set_logging(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, - 0 + 0, + awsRegionG }; S3ResponseHandler responseHandler = @@ -3634,11 +3871,12 @@ void set_logging(int argc, char **argv, int optindex) }; do { - S3_set_server_access_logging(&bucketContext, targetBucket, - targetPrefix, aclGrantCount, aclGrants, - 0, &responseHandler, 0); + S3_set_server_access_logging(&bucketContext, targetBucket, + targetPrefix, aclGrantCount, aclGrants, + 0, + timeoutMsG, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); - + if (statusG != S3StatusOK) { printError(); } @@ -3654,7 +3892,7 @@ int main(int argc, char **argv) // Parse args while (1) { int idx = 0; - int c = getopt_long(argc, argv, "vfhusr:", longOptionsG, &idx); + int c = getopt_long(argc, argv, "vfhusr:t:g:", longOptionsG, &idx); if (c == -1) { // End of options @@ -3682,11 +3920,24 @@ int main(int argc, char **argv) retriesG += *v - '0'; v++; } + } + break; + case 't': { + const char *v = optarg; + timeoutMsG = 0; + while (*v) { + timeoutMsG *= 10; + timeoutMsG += *v - '0'; + v++; + } + } break; case 'v': verifyPeerG = S3_INIT_VERIFY_PEER; break; - } + case 'g': + awsRegionG = strdup(optarg); + break; default: fprintf(stderr, "\nERROR: Unknown option: -%c\n", c); // Usage exit @@ -3701,7 +3952,7 @@ int main(int argc, char **argv) } const char *command = argv[optind++]; - + if (!strcmp(command, "help")) { fprintf(stdout, "\ns3 is a program for performing single requests " "to Amazon S3.\n"); @@ -3715,7 +3966,7 @@ int main(int argc, char **argv) } secretAccessKeyG = getenv("S3_SECRET_ACCESS_KEY"); if (!secretAccessKeyG) { - fprintf(stderr, + fprintf(stderr, "Missing environment variable: S3_SECRET_ACCESS_KEY\n"); return -1; } @@ -3731,7 +3982,7 @@ int main(int argc, char **argv) } else if (!strcmp(command, "delete")) { if (optind == argc) { - fprintf(stderr, + fprintf(stderr, "\nERROR: Missing parameter: bucket or bucket/key\n"); usageExit(stderr); } @@ -3771,6 +4022,12 @@ int main(int argc, char **argv) else if (!strcmp(command, "setacl")) { set_acl(argc, argv, optind); } + else if (!strcmp(command, "getlifecycle")) { + get_lifecycle(argc, argv, optind); + } + else if (!strcmp(command, "setlifecycle")) { + set_lifecycle(argc, argv, optind); + } else if (!strcmp(command, "getlogging")) { get_logging(argc, argv, optind); } diff --git a/src/service.c b/src/service.c index cc505d8..b6a1c10 100644 --- a/src/service.c +++ b/src/service.c @@ -1,13 +1,15 @@ /** ************************************************************************** * service.c - * + * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include @@ -34,7 +40,7 @@ typedef struct XmlCallbackData { SimpleXml simpleXml; - + S3ResponsePropertiesCallback *responsePropertiesCallback; S3ListServiceCallback *listServiceCallback; S3ResponseCompleteCallback *responseCompleteCallback; @@ -58,16 +64,16 @@ static S3Status xmlCallback(const char *elementPath, const char *data, if (!strcmp(elementPath, "ListAllMyBucketsResult/Owner/ID")) { string_buffer_append(cbData->ownerId, data, dataLen, fit); } - else if (!strcmp(elementPath, + else if (!strcmp(elementPath, "ListAllMyBucketsResult/Owner/DisplayName")) { string_buffer_append(cbData->ownerDisplayName, data, dataLen, fit); } - else if (!strcmp(elementPath, + else if (!strcmp(elementPath, "ListAllMyBucketsResult/Buckets/Bucket/Name")) { string_buffer_append(cbData->bucketName, data, dataLen, fit); } else if (!strcmp - (elementPath, + (elementPath, "ListAllMyBucketsResult/Buckets/Bucket/CreationDate")) { string_buffer_append(cbData->creationDate, data, dataLen, fit); } @@ -100,7 +106,7 @@ static S3Status propertiesCallback (const S3ResponseProperties *responseProperties, void *callbackData) { XmlCallbackData *cbData = (XmlCallbackData *) callbackData; - + return (*(cbData->responsePropertiesCallback)) (responseProperties, cbData->callbackData); } @@ -132,11 +138,13 @@ static void completeCallback(S3Status requestStatus, void S3_list_service(S3Protocol protocol, const char *accessKeyId, const char *secretAccessKey, const char *securityToken, - const char *hostName, S3RequestContext *requestContext, + const char *hostName, const char *authRegion, + S3RequestContext *requestContext, + int timeoutMs, const S3ListServiceHandler *handler, void *callbackData) { // Create and set up the callback data - XmlCallbackData *data = + XmlCallbackData *data = (XmlCallbackData *) malloc(sizeof(XmlCallbackData)); if (!data) { (*(handler->responseHandler.completeCallback)) @@ -156,7 +164,7 @@ void S3_list_service(S3Protocol protocol, const char *accessKeyId, string_buffer_initialize(data->ownerDisplayName); string_buffer_initialize(data->bucketName); string_buffer_initialize(data->creationDate); - + // Set up the RequestParams RequestParams params = { @@ -167,7 +175,8 @@ void S3_list_service(S3Protocol protocol, const char *accessKeyId, S3UriStylePath, // uriStyle accessKeyId, // accessKeyId secretAccessKey, // secretAccessKey - securityToken }, // securityToken + securityToken, // securityToken + authRegion }, // authRegion 0, // key 0, // queryParams 0, // subResource @@ -182,7 +191,8 @@ void S3_list_service(S3Protocol protocol, const char *accessKeyId, 0, // toS3CallbackTotalSize &dataCallback, // fromS3Callback &completeCallback, // completeCallback - data // callbackData + data, // callbackData + timeoutMs // timeoutMs }; // Perform the request diff --git a/src/service_access_logging.c b/src/service_access_logging.c index 28d0079..ce7ae6b 100644 --- a/src/service_access_logging.c +++ b/src/service_access_logging.c @@ -1,13 +1,15 @@ /** ************************************************************************** * server_access_logging.c - * + * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include @@ -60,10 +66,10 @@ static S3Status convertBlsXmlCallback(const char *elementPath, if (data) { if (!strcmp(elementPath, "BucketLoggingStatus/LoggingEnabled/" "TargetBucket")) { - caData->targetBucketReturnLen += + caData->targetBucketReturnLen += snprintf(&(caData->targetBucketReturn [caData->targetBucketReturnLen]), - 255 - caData->targetBucketReturnLen - 1, + 255 - caData->targetBucketReturnLen - 1, "%.*s", dataLen, data); if (caData->targetBucketReturnLen >= 255) { return S3StatusTargetBucketTooLong; @@ -71,10 +77,10 @@ static S3Status convertBlsXmlCallback(const char *elementPath, } else if (!strcmp(elementPath, "BucketLoggingStatus/LoggingEnabled/" "TargetPrefix")) { - caData->targetPrefixReturnLen += + caData->targetPrefixReturnLen += snprintf(&(caData->targetPrefixReturn [caData->targetPrefixReturnLen]), - 255 - caData->targetPrefixReturnLen - 1, + 255 - caData->targetPrefixReturnLen - 1, "%.*s", dataLen, data); if (caData->targetPrefixReturnLen >= 255) { return S3StatusTargetPrefixTooLong; @@ -142,7 +148,7 @@ static S3Status convertBlsXmlCallback(const char *elementPath, else if (caData->userId[0] && caData->userDisplayName[0]) { grant->granteeType = S3GranteeTypeCanonicalUser; strcpy(grant->grantee.canonicalUser.id, caData->userId); - strcpy(grant->grantee.canonicalUser.displayName, + strcpy(grant->grantee.canonicalUser.displayName, caData->userDisplayName); } else if (caData->groupUri[0]) { @@ -223,7 +229,7 @@ static S3Status convert_bls(char *blsXml, char *targetBucketReturn, S3Status status = simplexml_add(&simpleXml, blsXml, strlen(blsXml)); simplexml_deinitialize(&simpleXml); - + return status; } @@ -252,7 +258,7 @@ static S3Status getBlsPropertiesCallback (const S3ResponseProperties *responseProperties, void *callbackData) { GetBlsData *gsData = (GetBlsData *) callbackData; - + return (*(gsData->responsePropertiesCallback)) (responseProperties, gsData->callbackData); } @@ -266,12 +272,12 @@ static S3Status getBlsDataCallback(int bufferSize, const char *buffer, int fit; string_buffer_append(gsData->blsXmlDocument, buffer, bufferSize, fit); - + return fit ? S3StatusOK : S3StatusXmlDocumentTooLarge; } -static void getBlsCompleteCallback(S3Status requestStatus, +static void getBlsCompleteCallback(S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, void *callbackData) { @@ -281,7 +287,7 @@ static void getBlsCompleteCallback(S3Status requestStatus, // Parse the document requestStatus = convert_bls (gsData->blsXmlDocument, gsData->targetBucketReturn, - gsData->targetPrefixReturn, gsData->aclGrantCountReturn, + gsData->targetPrefixReturn, gsData->aclGrantCountReturn, gsData->aclGrants); } @@ -295,9 +301,10 @@ static void getBlsCompleteCallback(S3Status requestStatus, void S3_get_server_access_logging(const S3BucketContext *bucketContext, char *targetBucketReturn, char *targetPrefixReturn, - int *aclGrantCountReturn, + int *aclGrantCountReturn, S3AclGrant *aclGrants, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData) { @@ -329,7 +336,8 @@ void S3_get_server_access_logging(const S3BucketContext *bucketContext, bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId bucketContext->secretAccessKey, // secretAccessKey - bucketContext->securityToken }, // securityToken + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion 0, // key 0, // queryParams "logging", // subResource @@ -344,7 +352,8 @@ void S3_get_server_access_logging(const S3BucketContext *bucketContext, 0, // toS3CallbackTotalSize &getBlsDataCallback, // fromS3Callback &getBlsCompleteCallback, // completeCallback - gsData // callbackData + gsData, // callbackData + timeoutMs // timeoutMs }; // Perform the request @@ -357,7 +366,7 @@ void S3_get_server_access_logging(const S3BucketContext *bucketContext, static S3Status generateSalXmlDocument(const char *targetBucket, const char *targetPrefix, - int aclGrantCount, + int aclGrantCount, const S3AclGrant *aclGrants, int *xmlDocumentLenReturn, char *xmlDocument, @@ -381,7 +390,7 @@ static S3Status generateSalXmlDocument(const char *targetBucket, if (targetBucket && targetBucket[0]) { append("%s", targetBucket); - append("%s", + append("%s", targetPrefix ? targetPrefix : ""); if (aclGrantCount) { @@ -401,7 +410,7 @@ static S3Status generateSalXmlDocument(const char *targetBucket, case S3GranteeTypeCanonicalUser: append("CanonicalUser\">%s%s" "", - grant->grantee.canonicalUser.id, + grant->grantee.canonicalUser.id, grant->grantee.canonicalUser.displayName); break; default: // case S3GranteeTypeAllAwsUsers/S3GranteeTypeAllUsers: @@ -413,9 +422,9 @@ static S3Status generateSalXmlDocument(const char *targetBucket, append("%s", ((grant->permission == S3PermissionRead) ? "READ" : (grant->permission == S3PermissionWrite) ? "WRITE" : - (grant->permission == + (grant->permission == S3PermissionReadACP) ? "READ_ACP" : - (grant->permission == + (grant->permission == S3PermissionWriteACP) ? "WRITE_ACP" : "FULL_CONTROL")); } append("%s", ""); @@ -446,7 +455,7 @@ static S3Status setSalPropertiesCallback (const S3ResponseProperties *responseProperties, void *callbackData) { SetSalData *paData = (SetSalData *) callbackData; - + return (*(paData->responsePropertiesCallback)) (responseProperties, paData->callbackData); } @@ -456,11 +465,11 @@ static int setSalDataCallback(int bufferSize, char *buffer, void *callbackData) { SetSalData *paData = (SetSalData *) callbackData; - int remaining = (paData->salXmlDocumentLen - + int remaining = (paData->salXmlDocumentLen - paData->salXmlDocumentBytesWritten); int toCopy = bufferSize > remaining ? remaining : bufferSize; - + if (!toCopy) { return 0; } @@ -474,7 +483,7 @@ static int setSalDataCallback(int bufferSize, char *buffer, void *callbackData) } -static void setSalCompleteCallback(S3Status requestStatus, +static void setSalCompleteCallback(S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, void *callbackData) { @@ -488,10 +497,11 @@ static void setSalCompleteCallback(S3Status requestStatus, void S3_set_server_access_logging(const S3BucketContext *bucketContext, - const char *targetBucket, - const char *targetPrefix, int aclGrantCount, - const S3AclGrant *aclGrants, + const char *targetBucket, + const char *targetPrefix, int aclGrantCount, + const S3AclGrant *aclGrants, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData) { @@ -506,11 +516,11 @@ void S3_set_server_access_logging(const S3BucketContext *bucketContext, (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData); return; } - + // Convert aclGrants to XML document S3Status status = generateSalXmlDocument (targetBucket, targetPrefix, aclGrantCount, aclGrants, - &(data->salXmlDocumentLen), data->salXmlDocument, + &(data->salXmlDocumentLen), data->salXmlDocument, sizeof(data->salXmlDocument)); if (status != S3StatusOK) { free(data); @@ -534,7 +544,8 @@ void S3_set_server_access_logging(const S3BucketContext *bucketContext, bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId bucketContext->secretAccessKey, // secretAccessKey - bucketContext->securityToken }, // securityToken + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion 0, // key 0, // queryParams "logging", // subResource @@ -549,7 +560,8 @@ void S3_set_server_access_logging(const S3BucketContext *bucketContext, data->salXmlDocumentLen, // toS3CallbackTotalSize 0, // fromS3Callback &setSalCompleteCallback, // completeCallback - data // callbackData + data, // callbackData + timeoutMs // timeoutMs }; // Perform the request diff --git a/src/simplexml.c b/src/simplexml.c index 658ccce..4f26101 100644 --- a/src/simplexml.c +++ b/src/simplexml.c @@ -2,12 +2,14 @@ * simplexml.c * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include diff --git a/src/testsimplexml.c b/src/testsimplexml.c index 6fb7dee..941302f 100644 --- a/src/testsimplexml.c +++ b/src/testsimplexml.c @@ -2,12 +2,14 @@ * testsimplexml.c * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include diff --git a/src/util.c b/src/util.c index 590d037..7bdcf08 100644 --- a/src/util.c +++ b/src/util.c @@ -2,12 +2,14 @@ * util.c * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include @@ -53,8 +59,14 @@ static int checkString(const char *str, const char *format) return 1; } - -int urlEncode(char *dest, const char *src, int maxSrcSize) +/* + * Encode rules: + * 1. Every byte except: 'A'-'Z', 'a'-'z', '0'-'9', '-', '.', '_', and '~' + * 2. The space must be encoded as "%20" (and not as "+") + * 3. Letters in the hexadecimal value must be uppercase, for example "%1A" + * 4. Encode the forward slash character, '/', everywhere except in the object key name + */ +int urlEncode(char *dest, const char *src, int maxSrcSize, int encodeSlash) { static const char *hex = "0123456789ABCDEF"; @@ -67,9 +79,8 @@ int urlEncode(char *dest, const char *src, int maxSrcSize) } unsigned char c = *src; if (isalnum(c) || - (c == '-') || (c == '_') || (c == '.') || (c == '!') || - (c == '~') || (c == '*') || (c == '\'') || (c == '(') || - (c == ')') || (c == '/')) { + (c == '-') || (c == '_') || (c == '.') || + (c == '~') || (c == '/' && !encodeSlash)) { *dest++ = c; } else { @@ -120,7 +131,7 @@ int64_t parseIso8601Time(const char *str) str += 2; stm.tm_isdst = -1; - + int64_t ret = mktime(&stm); // Skip the millis @@ -131,7 +142,7 @@ int64_t parseIso8601Time(const char *str) str++; } } - + if (checkString(str, "-dd:dd") || checkString(str, "+dd:dd")) { int sign = (*str++ == '-') ? -1 : 1; int hours = nextnum(); @@ -164,393 +175,6 @@ uint64_t parseUnsignedInt(const char *str) } -int base64Encode(const unsigned char *in, int inLen, char *out) -{ - static const char *ENC = - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - - char *original_out = out; - - while (inLen) { - // first 6 bits of char 1 - *out++ = ENC[*in >> 2]; - if (!--inLen) { - // last 2 bits of char 1, 4 bits of 0 - *out++ = ENC[(*in & 0x3) << 4]; - *out++ = '='; - *out++ = '='; - break; - } - // last 2 bits of char 1, first 4 bits of char 2 - *out++ = ENC[((*in & 0x3) << 4) | (*(in + 1) >> 4)]; - in++; - if (!--inLen) { - // last 4 bits of char 2, 2 bits of 0 - *out++ = ENC[(*in & 0xF) << 2]; - *out++ = '='; - break; - } - // last 4 bits of char 2, first 2 bits of char 3 - *out++ = ENC[((*in & 0xF) << 2) | (*(in + 1) >> 6)]; - in++; - // last 6 bits of char 3 - *out++ = ENC[*in & 0x3F]; - in++, inLen--; - } - - return (out - original_out); -} - - -#define rol(value, bits) (((value) << (bits)) | ((value) >> (32 - (bits)))) - -#define blk0L(i) (block->l[i] = (rol(block->l[i], 24) & 0xFF00FF00) \ - | (rol(block->l[i], 8) & 0x00FF00FF)) - -#define blk0B(i) (block->l[i]) - -#define blk(i) (block->l[i & 15] = rol(block->l[(i + 13) & 15] ^ \ - block->l[(i + 8) & 15] ^ \ - block->l[(i + 2) & 15] ^ \ - block->l[i & 15], 1)) - -#define R0_L(v, w, x, y, z, i) \ - z += ((w & (x ^ y)) ^ y) + blk0L(i) + 0x5A827999 + rol(v, 5); \ - w = rol(w, 30); -#define R0_B(v, w, x, y, z, i) \ - z += ((w & (x ^ y)) ^ y) + blk0B(i) + 0x5A827999 + rol(v, 5); \ - w = rol(w, 30); -#define R1(v, w, x, y, z, i) \ - z += ((w & (x ^ y)) ^ y) + blk(i) + 0x5A827999 + rol(v, 5); \ - w = rol(w, 30); -#define R2(v, w, x, y, z, i) \ - z += (w ^ x ^ y) + blk(i) + 0x6ED9EBA1 + rol(v, 5); \ - w = rol(w, 30); -#define R3(v, w, x, y, z, i) \ - z += (((w | x) & y) | (w & x)) + blk(i) + 0x8F1BBCDC + rol(v, 5); \ - w = rol(w, 30); -#define R4(v, w, x, y, z, i) \ - z += (w ^ x ^ y) + blk(i) + 0xCA62C1D6 + rol(v, 5); \ - w = rol(w, 30); - -#define R0A_L(i) R0_L(a, b, c, d, e, i) -#define R0B_L(i) R0_L(b, c, d, e, a, i) -#define R0C_L(i) R0_L(c, d, e, a, b, i) -#define R0D_L(i) R0_L(d, e, a, b, c, i) -#define R0E_L(i) R0_L(e, a, b, c, d, i) - -#define R0A_B(i) R0_B(a, b, c, d, e, i) -#define R0B_B(i) R0_B(b, c, d, e, a, i) -#define R0C_B(i) R0_B(c, d, e, a, b, i) -#define R0D_B(i) R0_B(d, e, a, b, c, i) -#define R0E_B(i) R0_B(e, a, b, c, d, i) - -#define R1A(i) R1(a, b, c, d, e, i) -#define R1B(i) R1(b, c, d, e, a, i) -#define R1C(i) R1(c, d, e, a, b, i) -#define R1D(i) R1(d, e, a, b, c, i) -#define R1E(i) R1(e, a, b, c, d, i) - -#define R2A(i) R2(a, b, c, d, e, i) -#define R2B(i) R2(b, c, d, e, a, i) -#define R2C(i) R2(c, d, e, a, b, i) -#define R2D(i) R2(d, e, a, b, c, i) -#define R2E(i) R2(e, a, b, c, d, i) - -#define R3A(i) R3(a, b, c, d, e, i) -#define R3B(i) R3(b, c, d, e, a, i) -#define R3C(i) R3(c, d, e, a, b, i) -#define R3D(i) R3(d, e, a, b, c, i) -#define R3E(i) R3(e, a, b, c, d, i) - -#define R4A(i) R4(a, b, c, d, e, i) -#define R4B(i) R4(b, c, d, e, a, i) -#define R4C(i) R4(c, d, e, a, b, i) -#define R4D(i) R4(d, e, a, b, c, i) -#define R4E(i) R4(e, a, b, c, d, i) - - -static void SHA1_transform(uint32_t state[5], const unsigned char buffer[64]) -{ - uint32_t a, b, c, d, e; - - typedef union { - unsigned char c[64]; - uint32_t l[16]; - } u; - - unsigned char w[64]; - u *block = (u *) w; - - memcpy(block, buffer, 64); - - a = state[0]; - b = state[1]; - c = state[2]; - d = state[3]; - e = state[4]; - - static uint32_t endianness_indicator = 0x1; - if (((unsigned char *) &endianness_indicator)[0]) { - R0A_L( 0); - R0E_L( 1); R0D_L( 2); R0C_L( 3); R0B_L( 4); R0A_L( 5); - R0E_L( 6); R0D_L( 7); R0C_L( 8); R0B_L( 9); R0A_L(10); - R0E_L(11); R0D_L(12); R0C_L(13); R0B_L(14); R0A_L(15); - } - else { - R0A_B( 0); - R0E_B( 1); R0D_B( 2); R0C_B( 3); R0B_B( 4); R0A_B( 5); - R0E_B( 6); R0D_B( 7); R0C_B( 8); R0B_B( 9); R0A_B(10); - R0E_B(11); R0D_B(12); R0C_B(13); R0B_B(14); R0A_B(15); - } - R1E(16); R1D(17); R1C(18); R1B(19); R2A(20); - R2E(21); R2D(22); R2C(23); R2B(24); R2A(25); - R2E(26); R2D(27); R2C(28); R2B(29); R2A(30); - R2E(31); R2D(32); R2C(33); R2B(34); R2A(35); - R2E(36); R2D(37); R2C(38); R2B(39); R3A(40); - R3E(41); R3D(42); R3C(43); R3B(44); R3A(45); - R3E(46); R3D(47); R3C(48); R3B(49); R3A(50); - R3E(51); R3D(52); R3C(53); R3B(54); R3A(55); - R3E(56); R3D(57); R3C(58); R3B(59); R4A(60); - R4E(61); R4D(62); R4C(63); R4B(64); R4A(65); - R4E(66); R4D(67); R4C(68); R4B(69); R4A(70); - R4E(71); R4D(72); R4C(73); R4B(74); R4A(75); - R4E(76); R4D(77); R4C(78); R4B(79); - - state[0] += a; - state[1] += b; - state[2] += c; - state[3] += d; - state[4] += e; -} - - -typedef struct -{ - uint32_t state[5]; - uint32_t count[2]; - unsigned char buffer[64]; -} SHA1Context; - - -static void SHA1_init(SHA1Context *context) -{ - context->state[0] = 0x67452301; - context->state[1] = 0xEFCDAB89; - context->state[2] = 0x98BADCFE; - context->state[3] = 0x10325476; - context->state[4] = 0xC3D2E1F0; - context->count[0] = context->count[1] = 0; -} - - -static void SHA1_update(SHA1Context *context, const unsigned char *data, - unsigned int len) -{ - uint32_t i, j; - - j = (context->count[0] >> 3) & 63; - - if ((context->count[0] += len << 3) < (len << 3)) { - context->count[1]++; - } - - context->count[1] += (len >> 29); - - if ((j + len) > 63) { - memcpy(&(context->buffer[j]), data, (i = 64 - j)); - SHA1_transform(context->state, context->buffer); - for ( ; (i + 63) < len; i += 64) { - SHA1_transform(context->state, &(data[i])); - } - j = 0; - } - else { - i = 0; - } - - memcpy(&(context->buffer[j]), &(data[i]), len - i); -} - - -static void SHA1_final(unsigned char digest[20], SHA1Context *context) -{ - uint32_t i; - unsigned char finalcount[8]; - - for (i = 0; i < 8; i++) { - finalcount[i] = (unsigned char) - ((context->count[(i >= 4 ? 0 : 1)] >> - ((3 - (i & 3)) * 8)) & 255); - } - - SHA1_update(context, (unsigned char *) "\200", 1); - - while ((context->count[0] & 504) != 448) { - SHA1_update(context, (unsigned char *) "\0", 1); - } - - SHA1_update(context, finalcount, 8); - - for (i = 0; i < 20; i++) { - digest[i] = (unsigned char) - ((context->state[i >> 2] >> ((3 - (i & 3)) * 8)) & 255); - } - - memset(context->buffer, 0, 64); - memset(context->state, 0, 20); - memset(context->count, 0, 8); - memset(&finalcount, 0, 8); - - SHA1_transform(context->state, context->buffer); -} - - -// HMAC-SHA-1: -// -// K - is key padded with zeros to 512 bits -// m - is message -// OPAD - 0x5c5c5c... -// IPAD - 0x363636... -// -// HMAC(K,m) = SHA1((K ^ OPAD) . SHA1((K ^ IPAD) . m)) -void HMAC_SHA1(unsigned char hmac[20], const unsigned char *key, int key_len, - const unsigned char *message, int message_len) -{ - unsigned char kopad[64], kipad[64]; - int i; - - if (key_len > 64) { - key_len = 64; - } - - for (i = 0; i < key_len; i++) { - kopad[i] = key[i] ^ 0x5c; - kipad[i] = key[i] ^ 0x36; - } - - for ( ; i < 64; i++) { - kopad[i] = 0 ^ 0x5c; - kipad[i] = 0 ^ 0x36; - } - - unsigned char digest[20]; - - SHA1Context context; - - SHA1_init(&context); - SHA1_update(&context, kipad, 64); - SHA1_update(&context, message, message_len); - SHA1_final(digest, &context); - - SHA1_init(&context); - SHA1_update(&context, kopad, 64); - SHA1_update(&context, digest, 20); - SHA1_final(hmac, &context); -} - -#define rot(x,k) (((x) << (k)) | ((x) >> (32 - (k)))) - -uint64_t hash(const unsigned char *k, int length) -{ - uint32_t a, b, c; - - a = b = c = 0xdeadbeef + ((uint32_t) length); - - static uint32_t endianness_indicator = 0x1; - if (((unsigned char *) &endianness_indicator)[0]) { - while (length > 12) { - a += k[0]; - a += ((uint32_t) k[1]) << 8; - a += ((uint32_t) k[2]) << 16; - a += ((uint32_t) k[3]) << 24; - b += k[4]; - b += ((uint32_t) k[5]) << 8; - b += ((uint32_t) k[6]) << 16; - b += ((uint32_t) k[7]) << 24; - c += k[8]; - c += ((uint32_t) k[9]) << 8; - c += ((uint32_t) k[10]) << 16; - c += ((uint32_t) k[11]) << 24; - a -= c; a ^= rot(c, 4); c += b; - b -= a; b ^= rot(a, 6); a += c; - c -= b; c ^= rot(b, 8); b += a; - a -= c; a ^= rot(c, 16); c += b; - b -= a; b ^= rot(a, 19); a += c; - c -= b; c ^= rot(b, 4); b += a; - length -= 12; - k += 12; - } - - switch(length) { - case 12: c += ((uint32_t) k[11]) << 24; - case 11: c += ((uint32_t) k[10]) << 16; - case 10: c += ((uint32_t) k[9]) << 8; - case 9 : c += k[8]; - case 8 : b += ((uint32_t) k[7]) << 24; - case 7 : b += ((uint32_t) k[6]) << 16; - case 6 : b += ((uint32_t) k[5]) << 8; - case 5 : b += k[4]; - case 4 : a += ((uint32_t) k[3]) << 24; - case 3 : a += ((uint32_t) k[2]) << 16; - case 2 : a += ((uint32_t) k[1]) << 8; - case 1 : a += k[0]; break; - case 0 : goto end; - } - } - else { - while (length > 12) { - a += ((uint32_t) k[0]) << 24; - a += ((uint32_t) k[1]) << 16; - a += ((uint32_t) k[2]) << 8; - a += ((uint32_t) k[3]); - b += ((uint32_t) k[4]) << 24; - b += ((uint32_t) k[5]) << 16; - b += ((uint32_t) k[6]) << 8; - b += ((uint32_t) k[7]); - c += ((uint32_t) k[8]) << 24; - c += ((uint32_t) k[9]) << 16; - c += ((uint32_t) k[10]) << 8; - c += ((uint32_t) k[11]); - a -= c; a ^= rot(c, 4); c += b; - b -= a; b ^= rot(a, 6); a += c; - c -= b; c ^= rot(b, 8); b += a; - a -= c; a ^= rot(c, 16); c += b; - b -= a; b ^= rot(a, 19); a += c; - c -= b; c ^= rot(b, 4); b += a; - length -= 12; - k += 12; - } - - switch(length) { - case 12: c += k[11]; - case 11: c += ((uint32_t) k[10]) << 8; - case 10: c += ((uint32_t) k[9]) << 16; - case 9 : c += ((uint32_t) k[8]) << 24; - case 8 : b += k[7]; - case 7 : b += ((uint32_t) k[6]) << 8; - case 6 : b += ((uint32_t) k[5]) << 16; - case 5 : b += ((uint32_t) k[4]) << 24; - case 4 : a += k[3]; - case 3 : a += ((uint32_t) k[2]) << 8; - case 2 : a += ((uint32_t) k[1]) << 16; - case 1 : a += ((uint32_t) k[0]) << 24; break; - case 0 : goto end; - } - } - - c ^= b; c -= rot(b, 14); - a ^= c; a -= rot(c, 11); - b ^= a; b -= rot(a, 25); - c ^= b; c -= rot(b, 16); - a ^= c; a -= rot(c, 4); - b ^= a; b -= rot(a, 14); - c ^= b; c -= rot(b, 24); - - end: - return ((((uint64_t) c) << 32) | b); -} - int is_blank(char c) { return ((c == ' ') || (c == '\t')); diff --git a/test/test.sh b/test/test.sh index 4ebba93..413554b 100755 --- a/test/test.sh +++ b/test/test.sh @@ -28,157 +28,206 @@ if [ -z "$S3_COMMAND" ]; then S3_COMMAND=s3 fi +failures=0 + TEST_BUCKET=${TEST_BUCKET_PREFIX}.testbucket # Create the test bucket echo "$S3_COMMAND create $TEST_BUCKET" $S3_COMMAND create $TEST_BUCKET +failures=$(($failures + (($? == 0) ? 0 : 1))) # List to find it echo "$S3_COMMAND list | grep $TEST_BUCKET" $S3_COMMAND list | grep $TEST_BUCKET +failures=$(($failures + (($? == 0) ? 0 : 1))) # Test it echo "$S3_COMMAND test $TEST_BUCKET" $S3_COMMAND test $TEST_BUCKET +failures=$(($failures + (($? == 0) ? 0 : 1))) # List to ensure that it is empty echo "$S3_COMMAND list $TEST_BUCKET" $S3_COMMAND list $TEST_BUCKET +failures=$(($failures + (($? == 0) ? 0 : 1))) # Put some data rm -f seqdata seq 1 10000 > seqdata echo "$S3_COMMAND put $TEST_BUCKET/testkey filename=seqdata noStatus=1" $S3_COMMAND put $TEST_BUCKET/testkey filename=seqdata noStatus=1 +failures=$(($failures + (($? == 0) ? 0 : 1))) rm -f testkey # Get the data and make sure that it matches echo "$S3_COMMAND get $TEST_BUCKET/testkey filename=testkey" $S3_COMMAND get $TEST_BUCKET/testkey filename=testkey +failures=$(($failures + (($? == 0) ? 0 : 1))) diff seqdata testkey +failures=$(($failures + (($? == 0) ? 0 : 1))) rm -f seqdata testkey # Delete the file echo "$S3_COMMAND delete $TEST_BUCKET/testkey" $S3_COMMAND delete $TEST_BUCKET/testkey +failures=$(($failures + (($? == 0) ? 0 : 1))) # Remove the test bucket echo "$S3_COMMAND delete $TEST_BUCKET" $S3_COMMAND delete $TEST_BUCKET +failures=$(($failures + (($? == 0) ? 0 : 1))) # Make sure it's not there echo "$S3_COMMAND list | grep $TEST_BUCKET" $S3_COMMAND list | grep $TEST_BUCKET +failures=$(($failures + (($? == 1) ? 0 : 1))) # Now create it again echo "$S3_COMMAND create $TEST_BUCKET" $S3_COMMAND create $TEST_BUCKET +failures=$(($failures + (($? == 0) ? 0 : 1))) # Put 10 files in it for i in `seq 0 9`; do echo "echo \"Hello\" | $S3_COMMAND put $TEST_BUCKET/key_$i" echo "Hello" | $S3_COMMAND put $TEST_BUCKET/key_$i + failures=$(($failures + (($? == 0) ? 0 : 1))) done # List with all details echo "$S3_COMMAND list $TEST_BUCKET" $S3_COMMAND list $TEST_BUCKET +failures=$(($failures + (($? == 0) ? 0 : 1))) COPY_BUCKET=${TEST_BUCKET_PREFIX}.copybucket # Create another test bucket and copy a file into it echo "$S3_COMMAND create $COPY_BUCKET" $S3_COMMAND create $COPY_BUCKET -echo <> acl -Group Authenticated AWS Users READ +cat <> acl +Group Authenticated AWS Users READ EOF -echo <> acl -Group All Users READ_ACP +cat <> acl +Group All Users READ_ACP EOF echo "$S3_COMMAND setacl $TEST_BUCKET filename=acl" $S3_COMMAND setacl $TEST_BUCKET filename=acl +failures=$(($failures + (($? == 0) ? 0 : 1))) # Test to make sure that it worked rm -f acl_new echo "$S3_COMMAND getacl $TEST_BUCKET filename=acl_new" $S3_COMMAND getacl $TEST_BUCKET filename=acl_new -diff acl acl_new +failures=$(($failures + (($? == 0) ? 0 : 1))) +diff -B acl acl_new +failures=$(($failures + (($? == 0) ? 0 : 1))) rm -f acl acl_new # Get the key acl rm -f acl echo "$S3_COMMAND getacl $TEST_BUCKET/aclkey filename=acl" $S3_COMMAND getacl $TEST_BUCKET/aclkey filename=acl +failures=$(($failures + (($? == 0) ? 0 : 1))) # Add READ for all AWS users, and READ_ACP for everyone -echo <> acl -Group Authenticated AWS Users READ +cat <> acl +Group Authenticated AWS Users READ EOF -echo <> acl -Group All Users READ_ACP +cat <> acl +Group All Users READ_ACP EOF echo "$S3_COMMAND setacl $TEST_BUCKET/aclkey filename=acl" $S3_COMMAND setacl $TEST_BUCKET/aclkey filename=acl +failures=$(($failures + (($? == 0) ? 0 : 1))) # Test to make sure that it worked rm -f acl_new echo "$S3_COMMAND getacl $TEST_BUCKET/aclkey filename=acl_new" $S3_COMMAND getacl $TEST_BUCKET/aclkey filename=acl_new -diff acl acl_new +failures=$(($failures + (($? == 0) ? 0 : 1))) +diff -B acl acl_new +failures=$(($failures + (($? == 0) ? 0 : 1))) rm -f acl acl_new # Check multipart file upload (>15MB) dd if=/dev/zero of=mpfile bs=1024k count=30 echo "$S3_COMMAND put $TEST_BUCKET/mpfile filename=mpfile" $S3_COMMAND put $TEST_BUCKET/mpfile filename=mpfile +failures=$(($failures + (($? == 0) ? 0 : 1))) echo "$S3_COMMAND get $TEST_BUCKET/mpfile filename=mpfile.get" $S3_COMMAND get $TEST_BUCKET/mpfile filename=mpfile.get +failures=$(($failures + (($? == 0) ? 0 : 1))) diff mpfile mpfile.get +failures=$(($failures + (($? == 0) ? 0 : 1))) rm -f mpfile mpfile.get -# Remove the test file +# Remove the test files +echo "$S3_COMMAND delete $TEST_BUCKET/mpfile" +$S3_COMMAND delete $TEST_BUCKET/mpfile +failures=$(($failures + (($? == 0) ? 0 : 1))) echo "$S3_COMMAND delete $TEST_BUCKET/aclkey" $S3_COMMAND delete $TEST_BUCKET/aclkey +failures=$(($failures + (($? == 0) ? 0 : 1))) echo "$S3_COMMAND delete $TEST_BUCKET" $S3_COMMAND delete $TEST_BUCKET +failures=$(($failures + (($? == 0) ? 0 : 1))) + +if [ ${failures} = 0 ]; then + echo "all tests completed successfully" +else + echo "tests completed with ${failures} failures" +fi + +exit ${failures}