Skip to content

Commit 2079726

Browse files
taklwujojochuang
taklwu
authored and
jojochuang
committed
HADOOP-18671. Add recoverLease(), setSafeMode(), isFileClosed() as interfaces to hadoop-common (#5553)
The HDFS lease APIs have been replicated as interfaces in hadoop-common so other filesystems can also implement them. Applications which use the leasing APIs should migrate to the new interface where possible. Contributed by Stephen Wu (cherry picked from commit 0e46388) Conflicts: hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestViewDistributedFileSystem.java hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithAcl.java hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRetryCacheMetrics.java hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFSImageWithOrderedSnapshotDeletion.java hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOrderedSnapshotDeletion.java hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForErasureCodingPolicy.java Change-Id: I2ccd0b6780a86610df61d8528e681db0451e2e4c
1 parent 1756b49 commit 2079726

File tree

64 files changed

+843
-262
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

64 files changed

+843
-262
lines changed

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,5 +163,11 @@ private CommonPathCapabilities() {
163163
public static final String ETAGS_PRESERVED_IN_RENAME =
164164
"fs.capability.etags.preserved.in.rename";
165165

166+
/**
167+
* Does this Filesystem support lease recovery operations such as
168+
* {@link LeaseRecoverable#recoverLease(Path)} and {@link LeaseRecoverable#isFileClosed(Path)}}?
169+
* Value: {@value}.
170+
*/
171+
public static final String LEASE_RECOVERABLE = "fs.capability.lease.recoverable";
166172

167173
}

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LeaseRecoverable.java

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.fs;
20+
21+
import java.io.IOException;
22+
23+
/**
24+
* Whether the given Path of the FileSystem has the capability to perform lease recovery.
25+
*/
26+
public interface LeaseRecoverable {
27+
28+
/**
29+
* Start the lease recovery of a file.
30+
*
31+
* @param file path to a file.
32+
* @return true if the file is already closed, and it does not require lease recovery.
33+
* @throws IOException if an error occurs during lease recovery.
34+
* @throws UnsupportedOperationException if lease recovery is not supported by this filesystem.
35+
*/
36+
boolean recoverLease(Path file) throws IOException;
37+
38+
/**
39+
* Get the close status of a file.
40+
* @param file The string representation of the path to the file
41+
* @return return true if file is closed
42+
* @throws IOException If an I/O error occurred
43+
* @throws UnsupportedOperationException if isFileClosed is not supported by this filesystem.
44+
*/
45+
boolean isFileClosed(Path file) throws IOException;
46+
}

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/SafeMode.java

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.fs;
20+
21+
import java.io.IOException;
22+
23+
/**
24+
* Whether the given filesystem is in any status of safe mode.
25+
*/
26+
public interface SafeMode {
27+
28+
/**
29+
* Enter, leave, or get safe mode.
30+
*
31+
* @param action One of {@link SafeModeAction} LEAVE, ENTER, GET, FORCE_EXIT.
32+
* @throws IOException if set safe mode fails to proceed.
33+
* @return true if the action is successfully accepted, otherwise false means rejected.
34+
*/
35+
default boolean setSafeMode(SafeModeAction action) throws IOException {
36+
return setSafeMode(action, false);
37+
}
38+
39+
/**
40+
* Enter, leave, or get safe mode.
41+
*
42+
* @param action One of {@link SafeModeAction} LEAVE, ENTER, GET, FORCE_EXIT.
43+
* @param isChecked If true check only for Active metadata node / NameNode's status,
44+
* else check first metadata node / NameNode's status.
45+
* @throws IOException if set safe mode fails to proceed.
46+
* @return true if the action is successfully accepted, otherwise false means rejected.
47+
*/
48+
boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOException;
49+
50+
}

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/SafeModeAction.java

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.fs;
19+
20+
/**
21+
* An identical copy from org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction, that helps
22+
* the other file system implementation to define {@link SafeMode}.
23+
*/
24+
public enum SafeModeAction {
25+
/**
26+
* Starting entering into safe mode.
27+
*/
28+
ENTER,
29+
/**
30+
* Gracefully exit from safe mode.
31+
*/
32+
LEAVE,
33+
/**
34+
* Force Exit from safe mode.
35+
*/
36+
FORCE_EXIT,
37+
/**
38+
* Get the status of the safe mode.
39+
*/
40+
GET;
41+
}

‎hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,4 +41,6 @@ HDFS as these are commonly expected by Hadoop client applications.
4141
2. [Extending the specification and its tests](extending.html)
4242
1. [Uploading a file using Multiple Parts](multipartuploader.html)
4343
1. [IOStatistics](iostatistics.html)
44-
1. [openFile()](openfile.html).
44+
1. [openFile()](openfile.html)
45+
1. [SafeMode](safemode.html)
46+
1. [LeaseRecoverable](leaserecoverable.html)

‎hadoop-common-project/hadoop-common/src/site/markdown/filesystem/leaserecoverable.md

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
<!---
2+
Licensed under the Apache License, Version 2.0 (the "License");
3+
you may not use this file except in compliance with the License.
4+
You may obtain a copy of the License at
5+
6+
http://www.apache.org/licenses/LICENSE-2.0
7+
8+
Unless required by applicable law or agreed to in writing, software
9+
distributed under the License is distributed on an "AS IS" BASIS,
10+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
See the License for the specific language governing permissions and
12+
limitations under the License. See accompanying LICENSE file.
13+
-->
14+
15+
# <a name="LeaseRecoverable"></a> interface `LeaseRecoverable`
16+
17+
The `LeaseRecoverable` interface tells whether a given path of current filesystem can perform lease
18+
recovery for open file that a lease is not explicitly renewed or the client holding it goes away.
19+
20+
This interface should be implemented accordingly when necessary to any Filesystem that supports
21+
lease recovery, e.g. `DistributedFileSystem` (HDFS) and `ViewDistributedFileSystem`.
22+
23+
```java
24+
public interface LeaseRecoverable {
25+
boolean recoverLease(Path file) throws IOException;
26+
boolean isFileClosed(Path file) throws IOException;
27+
}
28+
```
29+
30+
There are two main functions of this interface, one performs lease recovery and another one
31+
verifies if a file has been closed.
32+
33+
### boolean recoverLease(Path file)
34+
35+
This function performs the lease recovery for the given file path, and it does not support
36+
directory path recovery.
37+
1. Return `true`, if the file has already closed, or does not require lease recovery.
38+
1. Return `false`, if the lease recovery is yet completed.
39+
1. Throw `IOException` if a directory path is given as input.
40+
41+
### boolean isFileClosed(Path file)
42+
43+
This function only checks if the give file path has been closed, and it does not support directory
44+
verification.
45+
1. Return `true`, if the file has been closed.
46+
1. Return `false`, if the file is still open.
47+
1. Throw `IOException` if a directory path is given as input.
48+
49+
### Path Capabilities SHOULD BE declared
50+
51+
If a filesystem supports `LeaseRecoverable`, it should return `true` to
52+
`PathCapabilities.hasPathCapability(path, "fs.capability.lease.recoverable")` for a given path.

‎hadoop-common-project/hadoop-common/src/site/markdown/filesystem/safemode.md

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
<!---
2+
Licensed under the Apache License, Version 2.0 (the "License");
3+
you may not use this file except in compliance with the License.
4+
You may obtain a copy of the License at
5+
6+
http://www.apache.org/licenses/LICENSE-2.0
7+
8+
Unless required by applicable law or agreed to in writing, software
9+
distributed under the License is distributed on an "AS IS" BASIS,
10+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
See the License for the specific language governing permissions and
12+
limitations under the License. See accompanying LICENSE file.
13+
-->
14+
15+
# <a name="SafeMode"></a> interface `SafeMode`
16+
17+
The `SafeMode` interface provides a way to perform safe mode actions and obtain the
18+
status after such actions performed to the `FileSystem`.
19+
20+
This is admin only interface, should be implemented accordingly when necessary to
21+
Filesystem that support safe mode, e.g. `DistributedFileSystem` (HDFS) and
22+
`ViewDistributedFileSystem`.
23+
24+
```java
25+
public interface SafeMode {
26+
default boolean setSafeMode(SafeModeAction action) throws IOException {
27+
return setSafeMode(action, false);
28+
}
29+
boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOException;
30+
}
31+
```
32+
33+
The goals of this interface is allow any file system implementation to share the
34+
same concept of safe mode with the following actions and states
35+
36+
### Safe mode actions
37+
1. `GET`, get the safe mode status of the file system.
38+
1. `ENTER`, enter the safe mode for the file system.
39+
1. `LEAVE`, exit safe mode for the file system gracefully.
40+
1. `FORCE_EXIT`, exit safe mode for the file system even if there is any ongoing data process.
41+
42+
### Safe mode states
43+
1. return true, when safe mode is on.
44+
1. return false, when safe mode is off, usually it's the result of safe mode actions
45+
with `GET`, `LEAVE`, `FORCE_EXIT`.

‎hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractLeaseRecoveryTest.java

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.fs.contract;
20+
21+
import java.io.FileNotFoundException;
22+
import java.io.IOException;
23+
24+
import org.assertj.core.api.Assertions;
25+
import org.junit.Test;
26+
27+
import org.apache.hadoop.fs.FileSystem;
28+
import org.apache.hadoop.fs.LeaseRecoverable;
29+
import org.apache.hadoop.fs.Path;
30+
import org.apache.hadoop.test.LambdaTestUtils;
31+
32+
import static org.apache.hadoop.fs.CommonPathCapabilities.LEASE_RECOVERABLE;
33+
34+
public abstract class AbstractContractLeaseRecoveryTest extends
35+
AbstractFSContractTestBase {
36+
37+
@Test
38+
public void testLeaseRecovery() throws Throwable {
39+
final Path path = methodPath();
40+
final FileSystem fs = getFileSystem();
41+
ContractTestUtils.touch(fs, path);
42+
LeaseRecoverable leaseRecoverableFs = verifyAndGetLeaseRecoverableInstance(fs, path);
43+
44+
Assertions.assertThat(leaseRecoverableFs.recoverLease(path))
45+
.describedAs("Issuing lease recovery on a closed file must be successful")
46+
.isTrue();
47+
48+
Assertions.assertThat(leaseRecoverableFs.isFileClosed(path))
49+
.describedAs("Get the isFileClose status on a closed file must be successful")
50+
.isTrue();
51+
}
52+
53+
@Test
54+
public void testLeaseRecoveryFileNotExist() throws Throwable {
55+
final Path path = new Path("notExist");
56+
final FileSystem fs = getFileSystem();
57+
LeaseRecoverable leaseRecoverableFs = verifyAndGetLeaseRecoverableInstance(fs, path);
58+
59+
LambdaTestUtils.intercept(FileNotFoundException.class, "File does not exist",
60+
() -> leaseRecoverableFs.recoverLease(path));
61+
62+
LambdaTestUtils.intercept(FileNotFoundException.class, "File does not exist",
63+
() -> leaseRecoverableFs.isFileClosed(path));
64+
}
65+
66+
@Test
67+
public void testLeaseRecoveryFileOnDirectory() throws Throwable {
68+
final Path path = methodPath();
69+
final FileSystem fs = getFileSystem();
70+
LeaseRecoverable leaseRecoverableFs = verifyAndGetLeaseRecoverableInstance(fs, path);
71+
final Path parentDirectory = path.getParent();
72+
73+
LambdaTestUtils.intercept(FileNotFoundException.class, "Path is not a file",
74+
() -> leaseRecoverableFs.recoverLease(parentDirectory));
75+
76+
LambdaTestUtils.intercept(FileNotFoundException.class, "Path is not a file",
77+
() -> leaseRecoverableFs.isFileClosed(parentDirectory));
78+
}
79+
80+
private LeaseRecoverable verifyAndGetLeaseRecoverableInstance(FileSystem fs, Path path)
81+
throws IOException {
82+
Assertions.assertThat(fs.hasPathCapability(path, LEASE_RECOVERABLE))
83+
.describedAs("path capability %s of %s", LEASE_RECOVERABLE, path)
84+
.isTrue();
85+
Assertions.assertThat(fs)
86+
.describedAs("filesystem %s", fs)
87+
.isInstanceOf(LeaseRecoverable.class);
88+
return (LeaseRecoverable) fs;
89+
}
90+
}

0 commit comments

Comments
 (0)