Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

COUCHDB-1654: Transparently update view signatures from <= 1.2.x.

Updates 1.2.x or earlier view files to 1.3.x or later view files
transparently, the first time the 1.2.x view file is opened by
1.3.x or later.

Here's how it works:

Before opening a view index,
If no matching index file is found in the new location:
 calculate the <= 1.2.x view signature
 if a file with that signature lives in the old location
   copy it to the new location with the new signature in the name.
Then proceed to open the view index as usual.
After opening, read its header.

If the header matches the <= 1.2.x style #index_header record:
  upgrade the header to the new #mrheader record
The next time the view is used, the new header is used.

If we crash after the rename, but before the header upgrade,
  the header upgrade is done on the next view opening.

If we crash between upgrading to the new header and writing
  that header to disk, we start with the old header again,
  do the upgrade and write to disk.

backport from Couchdb 1.3.x branch
  • Loading branch information...
commit 380fcb4d750b14d59cca054cc5ccd7e56e9da0c5 1 parent 7166e80
@benoitc benoitc authored
View
20 apps/couch_mrview/src/couch_mrview_index.erl
@@ -83,9 +83,29 @@ open(Db, State) ->
sig=Sig
} = State,
IndexFName = couch_mrview_util:index_file(DbName, Sig),
+
+ % If we are upgrading from <=1.2.x, we upgrade the view
+ % index file on the fly, avoiding an index reset.
+ %
+ % OldSig is `ok` if no upgrade happened.
+ %
+ % To remove suppport for 1.2.x auto-upgrades in the
+ % future, just remove the next line and the code
+ % between "upgrade code for <= 1.2.x" and
+ % "end upgrade code for <= 1.2.x" and the corresponding
+ % code in couch_mrview_util
+
+ OldSig = couch_mrview_util:maybe_update_index_file(State),
+
case couch_mrview_util:open_file(IndexFName) of
{ok, Fd} ->
case (catch couch_file:read_header(Fd)) of
+ % upgrade code for <= 1.2.x
+ {ok, {OldSig, Header}} ->
+ % Matching view signatures.
+ NewSt = couch_mrview_util:init_state(Db, Fd, State, Header),
+ {ok, NewSt#mrst{fd_monitor=erlang:monitor(process, Fd)}};
+ % end of upgrade code for <= 1.2.x
{ok, {Sig, Header}} ->
% Matching view signatures.
NewSt = couch_mrview_util:init_state(Db, Fd, State, Header),
View
100 apps/couch_mrview/src/couch_mrview_util.erl
@@ -25,6 +25,7 @@
-export([validate_args/1]).
-export([maybe_load_doc/3, maybe_load_doc/4]).
-export([to_seqkvs/2]).
+-export([maybe_update_index_file/1]).
-define(MOD, couch_mrview_index).
@@ -186,6 +187,19 @@ init_state(Db, Fd, #mrst{views=Views}=State, nil) ->
view_states=[{nil, nil, 0, 0} || _ <- Views]
},
init_state(Db, Fd, State, Header);
+% read <= 1.2.x header record and transpile it to >=1.3.x
+% header record
+init_state(Db, Fd, State, #index_header{
+ seq=Seq,
+ purge_seq=PurgeSeq,
+ id_btree_state=IdBtreeState,
+ view_states=ViewStates}) ->
+ init_state(Db, Fd, State, #mrheader{
+ seq=Seq,
+ purge_seq=PurgeSeq,
+ id_btree_state=IdBtreeState,
+ view_states=ViewStates
+ });
init_state(Db, Fd, State, Header) ->
#mrst{language=Lang, seq_indexed=SeqIndexed, views=Views} = State,
#mrheader{
@@ -764,6 +778,92 @@ mrverror(Mesg) ->
throw({query_parse_error, Mesg}).
+%% Updates 1.2.x or earlier view files to 1.3.x or later view files
+%% transparently, the first time the 1.2.x view file is opened by
+%% 1.3.x or later.
+%%
+%% Here's how it works:
+%%
+%% Before opening a view index,
+%% If no matching index file is found in the new location:
+%% calculate the <= 1.2.x view signature
+%% if a file with that signature lives in the old location
+%% rename it to the new location with the new signature in the name.
+%% Then proceed to open the view index as usual.
+%% After opening, read its header.
+%%
+%% If the header matches the <= 1.2.x style #index_header record:
+%% upgrade the header to the new #mrheader record
+%% The next time the view is used, the new header is used.
+%%
+%% If we crash after the rename, but before the header upgrade,
+%% the header upgrade is done on the next view opening.
+%%
+%% If we crash between upgrading to the new header and writing
+%% that header to disk, we start with the old header again,
+%% do the upgrade and write to disk.
+
+maybe_update_index_file(State) ->
+ DbName = State#mrst.db_name,
+ NewIndexFile = index_file(DbName, State#mrst.sig),
+ % open in read-only mode so we don't create
+ % the file if it doesn't exist.
+ case file:open(NewIndexFile, [read, raw]) of
+ {ok, Fd_Read} ->
+ % the new index file exists, there is nothing to do here.
+ file:close(Fd_Read);
+ _Error ->
+ update_index_file(State)
+ end.
+
+update_index_file(State) ->
+ Sig = sig_vsn_12x(State),
+ DbName = State#mrst.db_name,
+ FileName = couch_index_util:hexsig(Sig) ++ ".view",
+ IndexFile = couch_index_util:index_file("", DbName, FileName),
+
+ % If we have an old index, rename it to the new position.
+ case file:read_file_info(IndexFile) of
+ {ok, _FileInfo} ->
+ % Crash if the rename fails for any reason.
+ % If the target exists, e.g. the next request will find the
+ % new file and we are good. We might need to catch this
+ % further up to avoid a full server crash.
+ ?LOG_INFO("Attempting to update legacy view index file.", []),
+ NewIndexFile = index_file(DbName, State#mrst.sig),
+ ok = filelib:ensure_dir(NewIndexFile),
+ ok = file:rename(IndexFile, NewIndexFile),
+ ?LOG_INFO("Successfully updated legacy view index file.", []),
+ Sig;
+ _ ->
+ % Ignore missing index file
+ ok
+ end.
+
+sig_vsn_12x(State) ->
+ ViewInfo = [old_view_format(V) || V <- State#mrst.views],
+ SigData = case State#mrst.lib of
+ {[]} ->
+ {ViewInfo, State#mrst.language, State#mrst.design_opts};
+ _ ->
+ {ViewInfo, State#mrst.language, State#mrst.design_opts,
+ couch_index_util:sort_lib(State#mrst.lib)}
+ end,
+ couch_util:md5(term_to_binary(SigData)).
+
+old_view_format(View) ->
+{
+ view,
+ View#mrview.id_num,
+ View#mrview.map_names,
+ View#mrview.def,
+ View#mrview.btree,
+ View#mrview.reduce_funs,
+ View#mrview.options
+}.
+
+%% End of <= 1.2.x upgrade code.
+
to_seqkvs([], Acc) ->
lists:reverse(Acc);
to_seqkvs([{not_found, _} | Rest], Acc) ->
View
165 apps/couch_mrview/test/08-upgrade-legacy-view-files.t
@@ -0,0 +1,165 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+main(_) ->
+ test_util:init_code_path(),
+
+ etap:plan(8),
+ case (catch test()) of
+ ok ->
+ etap:end_tests();
+ Other ->
+ etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
+ etap:bail(Other)
+ end,
+ ok.
+
+
+test() ->
+ couch_server_sup:start_link(test_util:config_files()),
+
+ % commit sofort
+ ok = couch_config:set("query_server_config", "commit_freq", "0"),
+
+ test_upgrade(),
+
+ couch_server_sup:stop(),
+ ok.
+
+fixture_path() ->
+ test_util:srcdir() ++ "/test/etap/fixtures".
+
+old_db() ->
+ fixture_path() ++ "/" ++ old_db_name().
+
+old_db_name() ->
+ "test.couch".
+
+old_view() ->
+ fixture_path() ++ "/" ++ old_view_name().
+
+old_view_name() ->
+ "3b835456c235b1827e012e25666152f3.view".
+new_view_name() ->
+ "a1c5929f912aca32f13446122cc6ce50.view".
+
+couch_url() ->
+ "http://" ++ addr() ++ ":" ++ port().
+
+addr() ->
+ couch_config:get("httpd", "bind_address", "127.0.0.1").
+
+port() ->
+ integer_to_list(mochiweb_socket_server:get(couch_httpd, port)).
+
+
+% <= 1.2.x
+-record(index_header,
+ {seq=0,
+ purge_seq=0,
+ id_btree_state=nil,
+ view_states=nil
+ }).
+
+% >= 1.3.x
+-record(mrheader, {
+ seq=0,
+ purge_seq=0,
+ id_btree_state=nil,
+ view_states=nil
+}).
+
+ensure_header(File, MatchFun, Msg) ->
+ {ok, Fd} = couch_file:open(File),
+ {ok, {_Sig, Header}} = couch_file:read_header(Fd),
+ couch_file:close(Fd),
+ etap:fun_is(MatchFun, Header, "ensure " ++ Msg ++ " header for file: " ++ File).
+
+file_exists(File) ->
+ % open without creating
+ case file:open(File, [read, raw]) of
+ {ok, Fd_Read} ->
+ file:close(Fd_Read),
+ true;
+ _Error ->
+ false
+ end.
+
+cleanup() ->
+ DbDir = couch_config:get("couchdb", "database_dir"),
+ Files = [
+ DbDir ++ "/test.couch",
+ DbDir ++ "/.test_design/" ++ old_view_name(),
+ DbDir ++ "/.test_design/mrview/" ++ new_view_name()
+ ],
+ lists:foreach(fun(File) -> file:delete(File) end, Files),
+ etap:ok(true, "cleanup").
+
+test_upgrade() ->
+
+ cleanup(),
+
+ % copy old db file into db dir
+ DbDir = couch_config:get("couchdb", "database_dir"),
+ DbTarget = DbDir ++ "/" ++ old_db_name(),
+ filelib:ensure_dir(DbTarget),
+ file:copy(old_db(), DbTarget),
+
+ % copy old view file into view dir
+ ViewDir = couch_config:get("couchdb", "index_dir"),
+ ViewTarget = ViewDir ++ "/.test_design/" ++ old_view_name(),
+ filelib:ensure_dir(ViewTarget),
+ file:copy(old_view(), ViewTarget),
+
+ % ensure old header
+ ensure_header(ViewTarget, fun(#index_header{}) -> true; (_) -> false end, "old"),
+
+ % query view
+ ViewUrl = couch_url() ++ "/test/_design/test/_view/test",
+ {ok, Code, _Headers, Body} = test_util:request(ViewUrl, [], get),
+
+ % expect results
+ etap:is(Code, 200, "valid view result http status code"),
+ ExpectBody = <<"{\"total_rows\":2,\"offset\":0,\"rows\":[\r\n{\"id\":\"193f2f9c596ddc7ad326f7da470009ec\",\"key\":1,\"value\":null},\r\n{\"id\":\"193f2f9c596ddc7ad326f7da470012b6\",\"key\":2,\"value\":null}\r\n]}\n">>,
+ etap:is(Body, ExpectBody, "valid view result"),
+
+ % ensure old file gone.
+ etap:is(file_exists(ViewTarget), false, "ensure old file is gone"),
+
+ % ensure new header
+ NewViewFile = ViewDir ++ "/.test_design/mrview/" ++ new_view_name(),
+
+ % add doc(s)
+ test_util:request(
+ couch_url() ++ "/test/boo",
+ [{"Content-Type", "application/json"}],
+ put,
+ <<"{\"a\":3}">>),
+
+ % query again
+ {ok, Code2, _Headers2, Body2} = test_util:request(ViewUrl, [], get),
+
+ % expect results
+ etap:is(Code2, 200, "valid view result http status code"),
+ ExpectBody2 = <<"{\"total_rows\":3,\"offset\":0,\"rows\":[\r\n{\"id\":\"193f2f9c596ddc7ad326f7da470009ec\",\"key\":1,\"value\":null},\r\n{\"id\":\"193f2f9c596ddc7ad326f7da470012b6\",\"key\":2,\"value\":null},\r\n{\"id\":\"boo\",\"key\":3,\"value\":null}\r\n]}\n">>,
+ etap:is(Body2, ExpectBody2, "valid view result after doc add"),
+
+ % ensure no rebuild
+ % TBD no idea how to actually test this.
+
+ % ensure new header.
+ timer:sleep(1000),
+ ensure_header(NewViewFile, fun(#mrheader{}) -> true; (_) -> false end, "new"),
+
+ ok.
View
BIN  apps/couch_mrview/test/fixtures/3b835456c235b1827e012e25666152f3.view
Binary file not shown
View
BIN  apps/couch_mrview/test/fixtures/test.couch
Binary file not shown
Please sign in to comment.
Something went wrong with that request. Please try again.