Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HDDS-3231. Cleanup KeyManagerImpl #2961

Merged
merged 54 commits into from Jan 12, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
54 commits
Select commit Hold shift + click to select a range
c82a4ff
first tests working
Oct 28, 2021
59ae9bf
fixed block allocate tests
Oct 28, 2021
14d93ea
uncomment a test
Oct 29, 2021
0ab3257
simplified
Oct 29, 2021
b4b5b42
cleanup
Oct 29, 2021
12666c8
cleanup
Oct 29, 2021
fa128da
fixed precheck bug
Oct 29, 2021
bca8a80
KeyManagerUnit
Nov 6, 2021
939022a
om mocking
Nov 13, 2021
8733de1
added println
Nov 13, 2021
36f0633
testKeyManagerUnit passing
Dec 3, 2021
62a6330
fixed om
Dec 3, 2021
2d20e7c
fixed OM
Dec 3, 2021
73ca0a2
deleting service test
Dec 3, 2021
ba9b62a
added initKeyManager
Dec 3, 2021
20a52fd
working
Dec 3, 2021
5e5629b
cleanup
Dec 3, 2021
377e819
trash service
Dec 3, 2021
c43075b
parentAcl
Dec 3, 2021
fbf5218
volumeOwner
Dec 3, 2021
828ad76
removed km code
Dec 3, 2021
cc85e11
cleanup
Dec 3, 2021
77bd0be
OmTestUtils
Dec 3, 2021
ab1a891
OmTestWriteClient
Dec 3, 2021
fab7a76
restructured writeclient
Dec 3, 2021
0d29ff9
findbugs
Dec 4, 2021
d797083
cleanup
Dec 4, 2021
ceeaf37
cleanup
Dec 4, 2021
217bfcf
kmunit now uses OmTestWriteClient
Dec 4, 2021
3fa0419
cleanup
Dec 4, 2021
9a5188e
removed new createOm
Dec 4, 2021
dfd50df
fixed acl bug
Jan 2, 2022
6980f91
integrated OmTestWriteClient
Jan 3, 2022
ac33b03
fixed mocks
Jan 3, 2022
7c73f93
fixed the pipeline tests
Jan 3, 2022
7a53d3e
fixed testPrefixAclOps
Jan 3, 2022
2ce29e4
commented out failing access test
Jan 3, 2022
b44b27e
fixed versioning bug
Jan 3, 2022
01abaf4
removed directory at root test
Jan 3, 2022
929937b
cleanup
Jan 3, 2022
f0a805a
cleanup
Jan 3, 2022
d557352
added waitFor to keyDeleteing test
Jan 4, 2022
55e7e29
OmTestManagers
Jan 4, 2022
d6ec10d
fixed OmTestManagers locals
Jan 4, 2022
cdbdc67
added another constructor for OmTestManagers
Jan 4, 2022
752f510
cleanup
Jan 4, 2022
9dcc591
fixed OmTestManagers init
Jan 4, 2022
a4db56e
cleanup headers
Jan 4, 2022
834d7de
cleanup
Jan 4, 2022
2e3a1ed
cleanup
Jan 4, 2022
d9a9c7c
cleanup
Jan 4, 2022
38e133a
cleanup
Jan 4, 2022
eb84e84
trigger new CI check
Jan 5, 2022
527e84a
trigger new CI check
Jan 5, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view

Large diffs are not rendered by default.

Expand Up @@ -17,19 +17,12 @@
package org.apache.hadoop.ozone.om;

import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
import org.apache.hadoop.ozone.common.BlockGroup;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartCommitUploadPartInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteList;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.fs.OzoneManagerFS;
import org.apache.hadoop.hdds.utils.BackgroundService;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
Expand All @@ -55,44 +48,6 @@ public interface KeyManager extends OzoneManagerFS, IOzoneAcl {
*/
void stop() throws IOException;

/**
* After calling commit, the key will be made visible. There can be multiple
* open key writes in parallel (identified by client id). The most recently
* committed one will be the one visible.
*
* @param args the key to commit.
* @param clientID the client that is committing.
* @throws IOException
*/
void commitKey(OmKeyArgs args, long clientID) throws IOException;

/**
* A client calls this on an open key, to request to allocate a new block,
* and appended to the tail of current block list of the open client.
*
* @param args the key to append
* @param clientID the client requesting block.
* @param excludeList List of datanodes/containers to exclude during block
* allocation.
* @return the reference to the new block.
* @throws IOException
*/
OmKeyLocationInfo allocateBlock(OmKeyArgs args, long clientID,
ExcludeList excludeList) throws IOException;

/**
* Given the args of a key to put, write an open key entry to meta data.
*
* In case that the container creation or key write failed on
* DistributedStorageHandler, this key's metadata will still stay in OM.
* TODO garbage collect the open keys that never get closed
*
* @param args the args of the key provided by client.
* @return a OpenKeySession instance client uses to talk to container.
* @throws IOException
*/
OpenKeySession openKey(OmKeyArgs args) throws IOException;

/**
* Look up an existing key. Return the info of the key to client side, which
* DistributedStorageHandler will use to access the data on datanode.
Expand All @@ -105,26 +60,6 @@ OmKeyLocationInfo allocateBlock(OmKeyArgs args, long clientID,
*/
OmKeyInfo lookupKey(OmKeyArgs args, String clientAddress) throws IOException;

/**
* Renames an existing key within a bucket.
*
* @param args the args of the key provided by client.
* @param toKeyName New name to be used for the key
* @throws IOException if specified key doesn't exist or
* some other I/O errors while renaming the key.
*/
void renameKey(OmKeyArgs args, String toKeyName) throws IOException;

/**
* Deletes an object by an object key. The key will be immediately removed
* from OM namespace and become invisible to clients. The object data
* will be removed in async manner that might retain for some time.
*
* @param args the args of the key provided by client.
* @throws IOException if specified key doesn't exist or
* some other I/O errors while deleting an object.
*/
void deleteKey(OmKeyArgs args) throws IOException;

/**
* Returns a list of keys represented by {@link OmKeyInfo}
Expand Down Expand Up @@ -192,16 +127,6 @@ List<RepeatedOmKeyInfo> listTrash(String volumeName, String bucketName,
*/
List<String> getExpiredOpenKeys(int count) throws IOException;

/**
* Deletes a expired open key by its name. Called when a hanging key has been
* lingering for too long. Once called, the open key entries gets removed
* from OM mdata data.
*
* @param objectKeyName object key name with #open# prefix.
* @throws IOException if specified key doesn't exist or other I/O errors.
*/
void deleteExpiredOpenKey(String objectKeyName) throws IOException;

/**
* Returns the metadataManager.
* @return OMMetadataManager.
Expand All @@ -215,42 +140,6 @@ List<RepeatedOmKeyInfo> listTrash(String volumeName, String bucketName,
BackgroundService getDeletingService();


/**
* Initiate multipart upload for the specified key.
* @param keyArgs
* @return MultipartInfo
* @throws IOException
*/
OmMultipartInfo initiateMultipartUpload(OmKeyArgs keyArgs) throws IOException;

/**
* Commit Multipart upload part file.
* @param omKeyArgs
* @param clientID
* @return OmMultipartCommitUploadPartInfo
* @throws IOException
*/

OmMultipartCommitUploadPartInfo commitMultipartUploadPart(
OmKeyArgs omKeyArgs, long clientID) throws IOException;

/**
* Complete Multipart upload Request.
* @param omKeyArgs
* @param multipartUploadList
* @return OmMultipartUploadCompleteInfo
* @throws IOException
*/
OmMultipartUploadCompleteInfo completeMultipartUpload(OmKeyArgs omKeyArgs,
OmMultipartUploadCompleteList multipartUploadList) throws IOException;

/**
* Abort multipart upload request.
* @param omKeyArgs
* @throws IOException
*/
void abortMultipartUpload(OmKeyArgs omKeyArgs) throws IOException;

OmMultipartUploadList listMultipartUploads(String volumeName,
String bucketName, String prefix) throws OMException;

Expand Down