From d2179561d4e92c35d6609cb5f4aac109954033c8 Mon Sep 17 00:00:00 2001 From: lishuolin Date: Wed, 24 Apr 2024 15:42:28 +0800 Subject: [PATCH 1/2] enable new authorplan that can create raw password user. --- .../consensus/request/ConfigPhysicalPlan.java | 1 + .../request/ConfigPhysicalPlanType.java | 1 + .../confignode/persistence/AuthorInfo.java | 3 +++ .../persistence/AuthorInfoTest.java | 21 +++++++++++++++++++ .../auth/user/LocalFileUserManagerTest.java | 8 +++++++ .../auth/authorizer/BasicAuthorizer.java | 12 +++++++++-- .../commons/auth/authorizer/IAuthorizer.java | 2 ++ .../commons/auth/user/BasicUserManager.java | 16 +++++++++++--- .../iotdb/commons/auth/user/IUserManager.java | 3 ++- 9 files changed, 61 insertions(+), 6 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java index e2c6a5e963201..8ad8c9b2fd25a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java @@ -312,6 +312,7 @@ public static ConfigPhysicalPlan create(ByteBuffer buffer) throws IOException { case RevokeRole: case RevokeRoleFromUser: case UpdateUser: + case CreateUserWithRawPassword: plan = new AuthorPlan(configPhysicalPlanType); break; case ApplyConfigNode: diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java index 4e547304eb9bc..031518be9448d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java @@ -122,6 +122,7 @@ public enum ConfigPhysicalPlanType { ListUserRoles((short) 636), @Deprecated ListRoleUsers((short) 637), + CreateUserWithRawPassword((short) 638), /** Function. */ CreateFunction((short) 700), diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/AuthorInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/AuthorInfo.java index b026eb430533a..f0be01f2b9138 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/AuthorInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/AuthorInfo.java @@ -221,6 +221,9 @@ public TSStatus authorNonQuery(AuthorPlan authorPlan) { case CreateUser: authorizer.createUser(userName, password); break; + case CreateUserWithRawPassword: + authorizer.createUserWithRawPassword(userName, password); + break; case CreateRoleDep: AuthUtils.validateRolenamePre(roleName); authorizer.createRole(roleName); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/AuthorInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/AuthorInfoTest.java index 4c741d1c3612f..12bab54572a92 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/AuthorInfoTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/AuthorInfoTest.java @@ -32,6 +32,7 @@ import org.apache.iotdb.confignode.consensus.request.auth.AuthorPlan; import org.apache.iotdb.confignode.consensus.response.auth.PermissionInfoResp; import org.apache.iotdb.confignode.rpc.thrift.TCheckUserPrivilegesReq; +import org.apache.iotdb.confignode.rpc.thrift.TPermissionInfoResp; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.commons.io.FileUtils; @@ -1020,4 +1021,24 @@ public void testDepAuthorPlan() throws TException, AuthException, IllegalPathExc } } } + + @Test + public void createUserWithRawPassword() throws AuthException { + TSStatus status; + AuthorPlan authorPlan; + authorPlan = + new AuthorPlan( + ConfigPhysicalPlanType.CreateUserWithRawPassword, + "testuser", + "", + AuthUtils.encryptPassword("password"), + "", + new HashSet<>(), + false, + new ArrayList<>()); + status = authorInfo.authorNonQuery(authorPlan); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + TPermissionInfoResp result = authorInfo.login("testuser", "password"); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), result.getStatus().getCode()); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/user/LocalFileUserManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/user/LocalFileUserManagerTest.java index a837df4c654eb..43cd6df68dde0 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/user/LocalFileUserManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/user/LocalFileUserManagerTest.java @@ -183,4 +183,12 @@ public void test() throws AuthException, IllegalPathException { assertEquals(users[i].getName(), usernames.get(i + 1)); } } + + @Test + public void testCreateUserRawPassword() throws AuthException { + Assert.assertTrue( + manager.createUser("testRaw", AuthUtils.encryptPassword("password1"), true, false)); + User user = manager.getUser("testRaw"); + Assert.assertEquals(user.getPassword(), AuthUtils.encryptPassword("password1")); + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java index 4852ceee3e86f..e8359bcd070a2 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java @@ -125,7 +125,7 @@ public boolean login(String username, String password) throws AuthException { @Override public void createUser(String username, String password) throws AuthException { - if (!userManager.createUser(username, password, true)) { + if (!userManager.createUser(username, password, true, true)) { throw new AuthException( TSStatusCode.USER_ALREADY_EXIST, String.format("User %s already exists", username)); } @@ -133,7 +133,15 @@ public void createUser(String username, String password) throws AuthException { @Override public void createUserWithoutCheck(String username, String password) throws AuthException { - if (!userManager.createUser(username, password, false)) { + if (!userManager.createUser(username, password, false, true)) { + throw new AuthException( + TSStatusCode.USER_ALREADY_EXIST, String.format("User %s already exists", username)); + } + } + + @Override + public void createUserWithRawPassword(String username, String password) throws AuthException { + if (!userManager.createUser(username, password, true, false)) { throw new AuthException( TSStatusCode.USER_ALREADY_EXIST, String.format("User %s already exists", username)); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/IAuthorizer.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/IAuthorizer.java index 7546224bb7639..8b6d81a40f1b7 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/IAuthorizer.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/IAuthorizer.java @@ -284,5 +284,7 @@ boolean checkUserPrivileges(String username, PartialPath path, int privilegeId) */ void createUserWithoutCheck(String username, String password) throws AuthException; + void createUserWithRawPassword(String username, String password) throws AuthException; + void checkUserPathPrivilege(); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/user/BasicUserManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/user/BasicUserManager.java index aeeac635e24dd..b6445019d84e0 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/user/BasicUserManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/user/BasicUserManager.java @@ -103,6 +103,7 @@ private void initAdmin() throws AuthException { createUser( CommonDescriptor.getInstance().getConfig().getAdminName(), CommonDescriptor.getInstance().getConfig().getAdminPassword(), + true, true); setUserUseWaterMark(CommonDescriptor.getInstance().getConfig().getAdminName(), false); } @@ -138,11 +139,14 @@ public User getUser(String username) throws AuthException { } @Override - public boolean createUser(String username, String password, boolean validCheck) + public boolean createUser( + String username, String password, boolean validCheck, boolean enableEncrypt) throws AuthException { if (validCheck) { AuthUtils.validateUsername(username); - AuthUtils.validatePassword(password); + if (enableEncrypt) { + AuthUtils.validatePassword(password); + } } User user = getUser(username); @@ -151,7 +155,7 @@ public boolean createUser(String username, String password, boolean validCheck) } lock.writeLock(username); try { - user = new User(username, AuthUtils.encryptPassword(password)); + user = new User(username, enableEncrypt ? AuthUtils.encryptPassword(password) : password); userMap.put(username, user); return true; } finally { @@ -389,4 +393,10 @@ public void checkAndRefreshPathPri() { AuthUtils.checkAndRefreshPri(user); }); } + + @TestOnly + public boolean createUser(String username, String password, boolean validCheck) + throws AuthException { + return createUser(username, password, validCheck, true); + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/user/IUserManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/user/IUserManager.java index e1f830e6fa75b..d33f320a40644 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/user/IUserManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/user/IUserManager.java @@ -47,7 +47,8 @@ public interface IUserManager extends SnapshotProcessor { * @return True if the user is successfully created, false when the user already exists. * @throws AuthException if the given username or password is illegal. */ - boolean createUser(String username, String password, boolean validCheck) throws AuthException; + boolean createUser(String username, String password, boolean validCheck, boolean enableEncrypt) + throws AuthException; /** * Delete a user. From c9e64cc48e3fffedf91aeb3957d8e20c9fca717c Mon Sep 17 00:00:00 2001 From: lishuolin Date: Thu, 25 Apr 2024 09:28:42 +0800 Subject: [PATCH 2/2] fix pipe schema region loss tags. --- .../db/tools/schema/SRStatementGenerator.java | 91 ++++++++++++------- .../utils/SchemaRegionSnapshotParserTest.java | 86 ++++++++++++++++-- 2 files changed, 139 insertions(+), 38 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/schema/SRStatementGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/schema/SRStatementGenerator.java index b8b0363db1368..5ea6e86dc9c0c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/schema/SRStatementGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/schema/SRStatementGenerator.java @@ -280,8 +280,30 @@ public Statement visitMeasurementMNode( final CreateLogicalViewStatement stmt = new CreateLogicalViewStatement(); final LogicalViewSchema viewSchema = (LogicalViewSchema) node.getAsMeasurementMNode().getSchema(); - stmt.setTargetFullPaths(Collections.singletonList(path)); - stmt.setViewExpressions(Collections.singletonList(viewSchema.getExpression())); + if (viewSchema != null) { + stmt.setTargetFullPaths(Collections.singletonList(path)); + stmt.setViewExpressions(Collections.singletonList(viewSchema.getExpression())); + return stmt; + } + // if (node.getOffset() >= 0) { + // final AlterTimeSeriesStatement alterTimeSeriesStatement = new + // AlterTimeSeriesStatement(); + // alterTimeSeriesStatement.setPath(path); + // try { + // Pair, Map> tagsAndAttribute = + // getTagsAndAttributes(node.getOffset()); + // if (tagsAndAttribute != null) { + // alterTimeSeriesStatement.setTagsMap(tagsAndAttribute.left); + // alterTimeSeriesStatement.setAttributesMap(tagsAndAttribute.right); + // } + // } catch (IOException ioException) { + // lastExcept = ioException; + // LOGGER.warn( + // "Error when parse tag and attributes file of node path {}", path, + // ioException); + // } + // node.setOffset(0); + // } return stmt; } else { final CreateTimeSeriesStatement stmt = new CreateTimeSeriesStatement(); @@ -291,24 +313,18 @@ public Statement visitMeasurementMNode( stmt.setDataType(node.getDataType()); stmt.setEncoding(node.getAsMeasurementMNode().getSchema().getEncodingType()); if (node.getOffset() >= 0) { - if (tagFileChannel != null) { - try { - final ByteBuffer byteBuffer = - ByteBuffer.allocate(COMMON_CONFIG.getTagAttributeTotalSize()); - tagFileChannel.read(byteBuffer, node.getOffset()); - byteBuffer.flip(); - final Pair, Map> tagsAndAttributes = - new Pair<>( - ReadWriteIOUtils.readMap(byteBuffer), ReadWriteIOUtils.readMap(byteBuffer)); + try { + final Pair, Map> tagsAndAttributes = + getTagsAndAttributes(node.getOffset()); + if (tagsAndAttributes != null) { stmt.setTags(tagsAndAttributes.left); stmt.setAttributes(tagsAndAttributes.right); - } catch (IOException exception) { - lastExcept = exception; - LOGGER.warn("Error when parser tag and attributes files", exception); } - } else { - LOGGER.warn("Timeseries has attributes and tags but don't find tag file"); + } catch (IOException ioException) { + lastExcept = ioException; + LOGGER.warn("Error when parser tag and attributes files", ioException); } + node.setOffset(0); } return stmt; } @@ -338,30 +354,41 @@ private Statement genAlignedTimeseriesStatement(IMNode node, PartialPath path) { stmt.addEncoding(measurement.getAsMeasurementMNode().getSchema().getEncodingType()); stmt.addCompressor(measurement.getAsMeasurementMNode().getSchema().getCompressor()); if (measurement.getAsMeasurementMNode().getOffset() >= 0) { - if (tagFileChannel != null) { - try { - ByteBuffer byteBuffer = ByteBuffer.allocate(COMMON_CONFIG.getTagAttributeTotalSize()); - tagFileChannel.read(byteBuffer, measurement.getAsMeasurementMNode().getOffset()); - byteBuffer.flip(); - Pair, Map> tagsAndAttributes = - new Pair<>( - ReadWriteIOUtils.readMap(byteBuffer), ReadWriteIOUtils.readMap(byteBuffer)); + try { + Pair, Map> tagsAndAttributes = + getTagsAndAttributes(measurement.getAsMeasurementMNode().getOffset()); + if (tagsAndAttributes != null) { stmt.addAttributesList(tagsAndAttributes.right); stmt.addTagsList(tagsAndAttributes.left); - } catch (IOException exception) { - lastExcept = exception; - LOGGER.warn( - "Error when parse tag and attributes file of node path {}", - measurement.getPartialPath(), - exception); } - } else { - LOGGER.warn("Measurement has set attributes or tags, but not find snapshot files"); + } catch (IOException ioException) { + lastExcept = ioException; + LOGGER.warn( + "Error when parse tag and attributes file of node path {}", path, ioException); } + measurement.getAsMeasurementMNode().setOffset(0); + } else { + stmt.addAttributesList(null); + stmt.addTagsList(null); } } return stmt; } return null; } + + private Pair, Map> getTagsAndAttributes(long offset) + throws IOException { + if (tagFileChannel != null) { + ByteBuffer byteBuffer = ByteBuffer.allocate(COMMON_CONFIG.getTagAttributeTotalSize()); + tagFileChannel.read(byteBuffer, offset); + byteBuffer.flip(); + Pair, Map> tagsAndAttributes = + new Pair<>(ReadWriteIOUtils.readMap(byteBuffer), ReadWriteIOUtils.readMap(byteBuffer)); + return tagsAndAttributes; + } else { + LOGGER.warn("Measurement has set attributes or tags, but not find snapshot files"); + } + return null; + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaRegionSnapshotParserTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaRegionSnapshotParserTest.java index e893ab75cae24..8ff09c5008bca 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaRegionSnapshotParserTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaRegionSnapshotParserTest.java @@ -254,32 +254,43 @@ public void testAlignedTimeseriesTranslateSnapshot() throws Exception { ICreateAlignedTimeSeriesPlan plan = SchemaRegionWritePlanFactory.getCreateAlignedTimeSeriesPlan( new PartialPath("root.sg.t1.t2"), - Arrays.asList("s1", "s2"), - Arrays.asList(TSDataType.INT32, TSDataType.INT64), - Arrays.asList(TSEncoding.PLAIN, TSEncoding.RLE), - Arrays.asList(CompressionType.SNAPPY, CompressionType.LZ4), - Arrays.asList("alias1", "alias2"), + Arrays.asList("s1", "s2", "s3"), + Arrays.asList(TSDataType.INT32, TSDataType.INT64, TSDataType.BOOLEAN), + Arrays.asList(TSEncoding.PLAIN, TSEncoding.RLE, TSEncoding.PLAIN), + Arrays.asList( + CompressionType.SNAPPY, CompressionType.LZ4, CompressionType.UNCOMPRESSED), + Arrays.asList("alias1", "alias2", null), Arrays.asList( new HashMap() { { put("tag1", "t1"); + put("tag_1", "value2"); } }, new HashMap() { { put("tag2", "t2"); + put("tag_2", "t_2"); } - }), + }, + new HashMap<>()), // not all measurements have tag Arrays.asList( new HashMap() { { put("attr1", "a1"); + put("attr_1", "a_1"); } }, new HashMap() { { put("attr2", "a2"); } + }, + new HashMap() { + { + put("tag2", "t2"); + put("tag_2", "t_2"); + } })); schemaRegion.createAlignedTimeSeries(plan); File snapshotDir = new File(config.getSchemaDir() + File.separator + "snapshot"); @@ -305,6 +316,69 @@ public void testAlignedTimeseriesTranslateSnapshot() throws Exception { CreateAlignedTimeSeriesStatement createAlignedTimeSeriesStatement = (CreateAlignedTimeSeriesStatement) stmt; Assert.assertEquals(plan.getDevicePath(), createAlignedTimeSeriesStatement.getDevicePath()); + + Assert.assertEquals( + plan.getTagsList().size(), createAlignedTimeSeriesStatement.getTagsList().size()); + Assert.assertEquals( + plan.getAliasList().size(), createAlignedTimeSeriesStatement.getAliasList().size()); + Assert.assertEquals( + plan.getAttributesList().size(), + createAlignedTimeSeriesStatement.getAttributesList().size()); + Assert.assertEquals( + createAlignedTimeSeriesStatement.getMeasurements().size(), + createAlignedTimeSeriesStatement.getAttributesList().size()); + Assert.assertEquals( + createAlignedTimeSeriesStatement.getMeasurements().size(), + createAlignedTimeSeriesStatement.getTagsList().size()); + + Comparator> comp = + new Comparator>() { + @Override + public int compare(Map o1, Map o2) { + if (o1 == null && o2 == null) { + return 0; + } + if (o1 == null) { + return -1; + } + if (o2 == null) { + return 1; + } + return Integer.compare(o1.hashCode(), o2.hashCode()); + } + }; + + Comparator comp_str = + new Comparator() { + @Override + public int compare(String o1, String o2) { + if (o1 == null && o2 == null) { + return 0; + } + if (o1 == null) { + return -1; + } + if (o2 == null) { + return 1; + } + return Integer.compare(o1.hashCode(), o2.hashCode()); + } + }; + + Collections.sort(plan.getAliasList(), comp_str); + Collections.sort(createAlignedTimeSeriesStatement.getAliasList(), comp_str); + Collections.sort(plan.getAttributesList(), comp); + Collections.sort(createAlignedTimeSeriesStatement.getAttributesList(), comp); + Collections.sort(plan.getMeasurements(), comp_str); + Collections.sort(createAlignedTimeSeriesStatement.getMeasurements(), comp_str); + Collections.sort(plan.getTagsList(), comp); + Collections.sort(createAlignedTimeSeriesStatement.getTagsList(), comp); + Collections.sort(plan.getEncodings()); + Collections.sort(createAlignedTimeSeriesStatement.getEncodings()); + Collections.sort(plan.getCompressors()); + Collections.sort(createAlignedTimeSeriesStatement.getCompressors()); + Collections.sort(plan.getDataTypes()); + Collections.sort(createAlignedTimeSeriesStatement.getDataTypes()); Assert.assertEquals( plan.getMeasurements(), createAlignedTimeSeriesStatement.getMeasurements()); Assert.assertEquals(plan.getAliasList(), createAlignedTimeSeriesStatement.getAliasList());