aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorqiaoli <qiaoli@google.com>2023-06-27 17:26:19 +0000
committerqiaoli <qiaoli@google.com>2023-07-18 00:51:03 +0000
commit3b849307dbdfc75c68ce8918d0a5663c13fab379 (patch)
treea16bd6de61fe6a60083eefd8b3dbb191e9f2ceab
parent9e36da1f564c3b5a6c0c7565d0dc423c5e694779 (diff)
downloadfederated-compute-3b849307dbdfc75c68ce8918d0a5663c13fab379.tar.gz
Add tensoflow filesytem support file descriptor
Bug: 281758731 Test: atest Change-Id: I19d6d2deb8a8a66d5ef66c6a1640b867f3434b80
-rw-r--r--Android.bp14
-rw-r--r--PREUPLOAD.cfg5
-rw-r--r--fcp/client/engine/example_query_plan_engine_test.cc12
-rw-r--r--fcp/client/engine/tflite_plan_engine_test.cc14
-rw-r--r--fcp/client/fcp_runner.cc2
-rw-r--r--fcp/tensorflow/file_descriptor_filesystem.cc250
-rw-r--r--fcp/tensorflow/file_descriptor_filesystem.h72
-rw-r--r--fcp/tensorflow/file_descriptor_filesystem_test.cc203
8 files changed, 554 insertions, 18 deletions
diff --git a/Android.bp b/Android.bp
index 067f3de..cb41fc7 100644
--- a/Android.bp
+++ b/Android.bp
@@ -87,7 +87,6 @@ cc_library {
},
static_libs: [
"tensorflow_core_proto_cpp_lite",
- "libprotobuf-cpp-lite-ndk",
],
shared_libs: [
"liblog",
@@ -120,12 +119,13 @@ cc_library_static {
"fcp/client/engine/tf_*.cc",
"fcp/tensorflow/tf_session.cc",
],
- static_libs:[
+ static_libs: [
"federated-compute-cc-proto-lite",
],
whole_static_libs: [
"libtflite_flex_delegate",
],
+ export_include_dirs: ["."],
header_libs: [
"flatbuffer_headers",
"libeigen",
@@ -145,7 +145,6 @@ cc_library_static {
"-Wno-defaulted-function-deleted",
"-Wno-deprecated-declarations",
],
- export_include_dirs: ["."],
stl: "libc++_static",
sdk_version: "current",
apex_available: ["com.android.ondevicepersonalization"],
@@ -156,7 +155,7 @@ filegroup {
name: "fcp_native_wrapper",
srcs: ["fcp/java_src/main/java/com/google/fcp/client/CallFromNativeWrapper.java"],
visibility: [
- "//packages/modules/OnDevicePersonalization:__subpackages__"
+ "//packages/modules/OnDevicePersonalization:__subpackages__",
],
}
@@ -169,7 +168,7 @@ filegroup {
"fcp/testdata/federation_test_select_checkpoints.pb",
],
visibility: [
- "//packages/modules/OnDevicePersonalization:__subpackages__"
+ "//packages/modules/OnDevicePersonalization:__subpackages__",
],
}
@@ -177,6 +176,8 @@ cc_test {
name: "fcp_plan_engine_test",
srcs: [
"fcp/client/engine/tflite_plan_engine_test.cc",
+ "fcp/client/engine/example_query_plan_engine_test.cc",
+ "fcp/tensorflow/file_descriptor_filesystem_test.cc",
"fcp/client/test_helpers.cc",
],
test_suites: [
@@ -188,6 +189,7 @@ cc_test {
"libbase_ndk",
"libprotobuf-cpp-lite-ndk",
"liblog",
+ "tensorflow_abseil",
"libc++fs", // used by filesystem
],
whole_static_libs: [
@@ -212,4 +214,4 @@ cc_test {
"-Wno-ignored-qualifiers",
"-Wno-missing-field-initializers",
],
-} \ No newline at end of file
+}
diff --git a/PREUPLOAD.cfg b/PREUPLOAD.cfg
new file mode 100644
index 0000000..e33b85d
--- /dev/null
+++ b/PREUPLOAD.cfg
@@ -0,0 +1,5 @@
+[Builtin Hooks]
+bpfmt = true
+
+[Hook Scripts]
+checkstyle_hook = ${REPO_ROOT}/prebuilts/checkstyle/checkstyle.py --sha ${PREUPLOAD_COMMIT} \ No newline at end of file
diff --git a/fcp/client/engine/example_query_plan_engine_test.cc b/fcp/client/engine/example_query_plan_engine_test.cc
index dbad82d..638e861 100644
--- a/fcp/client/engine/example_query_plan_engine_test.cc
+++ b/fcp/client/engine/example_query_plan_engine_test.cc
@@ -25,8 +25,6 @@
#include <utility>
#include <vector>
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
#include "absl/container/flat_hash_map.h"
#include "absl/status/status.h"
#include "absl/status/statusor.h"
@@ -36,7 +34,8 @@
#include "fcp/client/example_query_result.pb.h"
#include "fcp/client/test_helpers.h"
#include "fcp/protos/plan.pb.h"
-#include "fcp/testing/testing.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
#include "tensorflow/c/checkpoint_reader.h"
#include "tensorflow/c/tf_status.h"
#include "tensorflow/c/tf_status_helper.h"
@@ -268,7 +267,7 @@ TEST_F(ExampleQueryPlanEngineTest, PlanSucceeds) {
EXPECT_THAT(result.outcome, PlanOutcome::kSuccess);
auto tensors = ReadTensors(output_checkpoint_filename_);
- ASSERT_OK(tensors);
+ // ASSERT_OK(tensors);
tf::Tensor int_tensor = tensors.value()[kOutputIntTensorName];
ASSERT_EQ(int_tensor.shape(), tf::TensorShape({2}));
ASSERT_EQ(int_tensor.dtype(), tf::DT_INT64);
@@ -360,7 +359,8 @@ TEST_F(ExampleQueryPlanEngineTest, MultipleQueries) {
EXPECT_THAT(result.outcome, PlanOutcome::kSuccess);
auto tensors = ReadTensors(output_checkpoint_filename_);
- ASSERT_OK(tensors);
+ // ASSERT_OK is not supported in AOSP.
+ // ASSERT_OK(tensors);
tf::Tensor int_tensor = tensors.value()[kOutputIntTensorName];
ASSERT_EQ(int_tensor.shape(), tf::TensorShape({2}));
ASSERT_EQ(int_tensor.dtype(), tf::DT_INT64);
@@ -544,4 +544,4 @@ TEST_F(ExampleQueryPlanEngineTest, InvalidExampleQueryResultFormat) {
} // anonymous namespace
} // namespace engine
} // namespace client
-} // namespace fcp
+} // namespace fcp \ No newline at end of file
diff --git a/fcp/client/engine/tflite_plan_engine_test.cc b/fcp/client/engine/tflite_plan_engine_test.cc
index 8e5319e..8fffa0e 100644
--- a/fcp/client/engine/tflite_plan_engine_test.cc
+++ b/fcp/client/engine/tflite_plan_engine_test.cc
@@ -122,15 +122,17 @@ class TfLitePlanEngineTest : public testing::Test {
FederatedComputeIORouter io_router =
client_only_plan_.phase().federated_compute();
if (!io_router.input_filepath_tensor_name().empty()) {
- (*inputs_)[io_router.input_filepath_tensor_name()] =
- checkpoint_input_filename_;
+ (*inputs_)[io_router.input_filepath_tensor_name()] = checkpoint_input_fd_;
}
checkpoint_output_filename_ =
files_impl_.CreateTempFile("output", ".ckp").value();
ASSERT_EQ(std::filesystem::file_size(checkpoint_output_filename_), 0);
+ int fd = open(checkpoint_output_filename_.c_str(), O_WRONLY);
+ ASSERT_NE(-1, fd);
+ checkpoint_output_fd_ = absl::StrCat("fd:///", fd);
if (!io_router.output_filepath_tensor_name().empty()) {
(*inputs_)[io_router.output_filepath_tensor_name()] =
- checkpoint_output_filename_;
+ checkpoint_output_fd_;
}
for (const auto& tensor_spec :
@@ -146,6 +148,9 @@ class TfLitePlanEngineTest : public testing::Test {
client_only_plan_ = std::move(artifacts->plan);
dataset_ = std::move(artifacts->dataset);
checkpoint_input_filename_ = artifacts->checkpoint_filepath;
+ int fd = open(checkpoint_input_filename_.c_str(), O_RDONLY);
+ ASSERT_NE(-1, fd);
+ checkpoint_input_fd_ = absl::StrCat("fd:///", fd);
}
void ComputeDatasetStats(const std::string& collection_uri) {
@@ -177,6 +182,8 @@ class TfLitePlanEngineTest : public testing::Test {
Dataset dataset_;
std::string checkpoint_input_filename_;
std::string checkpoint_output_filename_;
+ std::string checkpoint_input_fd_;
+ std::string checkpoint_output_fd_;
int num_examples_ = 0;
int example_bytes_ = 0;
@@ -209,7 +216,6 @@ TEST_F(TfLitePlanEngineTest, SimpleAggPlanSucceeds) {
engine::PlanResult result = plan_engine.RunPlan(
client_only_plan_.phase().tensorflow_spec(),
client_only_plan_.tflite_graph(), std::move(inputs_), output_names_);
- FCP_LOG(INFO) << "**** plan result " << result.original_status;
EXPECT_THAT(result.outcome, PlanOutcome::kSuccess);
EXPECT_THAT(result.output_tensors.size(), 0);
diff --git a/fcp/client/fcp_runner.cc b/fcp/client/fcp_runner.cc
index 4736b6a..8b645d5 100644
--- a/fcp/client/fcp_runner.cc
+++ b/fcp/client/fcp_runner.cc
@@ -280,8 +280,6 @@ absl::StatusOr<FLRunnerResult> RunFederatedComputation(
fl_runner_result.set_contribution_result(FLRunnerResult::SUCCESS);
} else {
fl_runner_result.set_contribution_result(FLRunnerResult::FAIL);
- std::string error_message = std::string{
- plan_result_and_checkpoint_file.plan_result.original_status.message()};
}
return fl_runner_result;
}
diff --git a/fcp/tensorflow/file_descriptor_filesystem.cc b/fcp/tensorflow/file_descriptor_filesystem.cc
new file mode 100644
index 0000000..1659839
--- /dev/null
+++ b/fcp/tensorflow/file_descriptor_filesystem.cc
@@ -0,0 +1,250 @@
+#include "fcp/tensorflow/file_descriptor_filesystem.h"
+
+#include <errno.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include <algorithm>
+#include <memory>
+#include <vector>
+
+#include "absl/memory/memory.h"
+#include "absl/strings/numbers.h"
+#include "absl/strings/str_cat.h"
+#include "absl/strings/string_view.h"
+#include "absl/strings/strip.h"
+#include "fcp/base/monitoring.h"
+#include "tensorflow/core/platform/env.h"
+
+namespace tensorflow {
+
+namespace fcp {
+
+using ::tensorflow::Status;
+
+static constexpr char kFdFilesystemPrefix[] = "fd:///";
+static constexpr size_t kMaxWriteChunkSize = 64u * 1024; // 64KB
+
+namespace {
+// Copied from base implementation in
+// //external/tensorflow/tensorflow/tsl/platform/default/posix_file_system.cc
+absl::Status ReadBytesFromFd(int fd, uint64_t offset, size_t n,
+ absl::string_view* result, char* scratch) {
+ absl::Status s;
+ char* dst = scratch;
+ while (n > 0 && s.ok()) {
+ ssize_t r = pread(fd, dst, n, static_cast<off_t>(offset));
+ if (r > 0) {
+ dst += r;
+ n -= r;
+ offset += r;
+ } else if (r == 0) {
+ s = absl::OutOfRangeError(absl::StrCat(
+ "Read fewer bytes than requested. Total read bytes ", offset));
+ } else if (errno == EINTR || errno == EAGAIN) {
+ // Retry
+ } else {
+ s = absl::UnknownError(absl::StrCat("Failed to read: errno ", errno));
+ }
+ }
+ *result = absl::string_view(scratch, dst - scratch);
+ return s;
+}
+
+class FdRandomAccessFile : public RandomAccessFile {
+ public:
+ explicit FdRandomAccessFile(int fd) : fd_(fd) {}
+
+ ~FdRandomAccessFile() override { close(fd_); }
+
+ Status Read(uint64 offset, size_t n, StringPiece* result,
+ char* scratch) const override {
+ absl::string_view sv;
+ absl::Status s = ReadBytesFromFd(fd_, offset, n, &sv, scratch);
+ if (s.ok()) {
+ *result = StringPiece(sv.data(), sv.size());
+ return Status();
+ } else {
+ return Status(static_cast<tensorflow::errors::Code>(s.code()),
+ s.message());
+ }
+ }
+
+ private:
+ int fd_;
+};
+
+class FdWritableFile : public WritableFile {
+ public:
+ explicit FdWritableFile(int fd) : fd_(fd) {}
+
+ ~FdWritableFile() override { Close(); }
+
+ Status Append(StringPiece data) override {
+ return Write(data.data(), data.size());
+ }
+
+ Status Close() override {
+ if (has_closed_) return Status();
+ close(fd_);
+ has_closed_ = true;
+ return Status();
+ }
+ Status Flush() override { return Status(); }
+ Status Sync() override { return Status(); }
+
+ private:
+ Status Write(const void* data, size_t data_size) {
+ size_t write_len = data_size;
+ do {
+ size_t chunk_size = std::min<size_t>(write_len, kMaxWriteChunkSize);
+ ssize_t wrote = write(fd_, data, chunk_size);
+ if (wrote < 0) {
+ return Status(tensorflow::error::Code::UNKNOWN,
+ absl::StrCat("Failed to write: ", errno));
+ }
+ data = static_cast<const uint8_t*>(data) + wrote;
+ write_len -= wrote;
+ } while (write_len > 0);
+ return Status();
+ }
+ int fd_;
+ bool has_closed_ = false;
+};
+
+// Gets the file descriptor in the URI.
+Status GetFd(absl::string_view fname, int* result) {
+ // Consume scheme and empty authority (fd:///)
+ if (!absl::ConsumePrefix(&fname, kFdFilesystemPrefix)) {
+ return errors::InvalidArgument("Bad uri: ", fname);
+ }
+
+ // Try to parse remainder of path as an integer fd
+ if (!absl::SimpleAtoi(fname, result)) {
+ return errors::InvalidArgument("Bad path: ", fname);
+ }
+
+ return OkStatus();
+}
+
+} // anonymous namespace
+
+Status FileDescriptorFileSystem::NewRandomAccessFile(
+ const string& filename, std::unique_ptr<RandomAccessFile>* result) {
+ int fd;
+ TF_RETURN_IF_ERROR(GetFd(filename, &fd));
+ FileStatistics stat;
+ TF_RETURN_IF_ERROR(Stat(filename, &stat)); // check against directory FD
+
+ int dup_fd = dup(fd);
+ if (dup_fd == -1) {
+ return errors::Unknown("Failed to dup: errno ", errno);
+ }
+
+ *result = std::make_unique<FdRandomAccessFile>(dup_fd);
+ return OkStatus();
+}
+
+Status FileDescriptorFileSystem::GetMatchingPaths(
+ const string& pattern, std::vector<string>* results) {
+ results->clear();
+ FileStatistics statistics;
+ if (Stat(pattern, &statistics).ok()) {
+ results->push_back(pattern);
+ }
+ return OkStatus();
+}
+
+Status FileDescriptorFileSystem::Stat(const string& fname,
+ FileStatistics* stats) {
+ if (stats == nullptr) {
+ return errors::InvalidArgument("FileStatistics pointer must not be NULL");
+ }
+
+ int fd;
+ TF_RETURN_IF_ERROR(GetFd(fname, &fd));
+
+ struct stat st;
+ if (fstat(fd, &st) == -1) {
+ return errors::Unknown("Failed to fstat: errno ", errno);
+ }
+
+ if (S_ISDIR(st.st_mode)) {
+ return errors::NotFound("File not found: is a directory");
+ }
+ stats->length = st.st_size;
+ stats->mtime_nsec = st.st_mtime * 1e9;
+ stats->is_directory = S_ISDIR(st.st_mode);
+
+ return OkStatus();
+}
+
+Status FileDescriptorFileSystem::GetFileSize(const string& fname,
+ uint64* size) {
+ FileStatistics stat;
+ TF_RETURN_IF_ERROR(Stat(fname, &stat));
+ *size = stat.length;
+ return OkStatus();
+}
+
+Status FileDescriptorFileSystem::NewReadOnlyMemoryRegionFromFile(
+ const string& filename, std::unique_ptr<ReadOnlyMemoryRegion>* result) {
+ return errors::Unimplemented("Not implemented by the fd filesystem");
+}
+
+Status FileDescriptorFileSystem::FileExists(const string& fname) {
+ return errors::Unimplemented("Not implemented by the fd filesystem");
+}
+
+Status FileDescriptorFileSystem::GetChildren(const string& dir,
+ std::vector<string>* r) {
+ return errors::Unimplemented("fd filesystem is non-hierarchical");
+}
+
+Status FileDescriptorFileSystem::NewWritableFile(
+ const string& fname, std::unique_ptr<WritableFile>* result) {
+ int fd;
+ TF_RETURN_IF_ERROR(GetFd(fname, &fd));
+ FileStatistics stat;
+ TF_RETURN_IF_ERROR(Stat(fname, &stat)); // check against directory FD
+
+ int dup_fd = dup(fd);
+ if (dup_fd == -1) {
+ return errors::Unknown("Failed to dup: errno ", errno);
+ }
+
+ *result = std::make_unique<FdWritableFile>(dup_fd);
+ return OkStatus();
+}
+
+Status FileDescriptorFileSystem::NewAppendableFile(
+ const string& fname, std::unique_ptr<WritableFile>* result) {
+ return errors::Unimplemented("Not implemented by the fd filesystem");
+}
+
+Status FileDescriptorFileSystem::DeleteFile(const string& f) {
+ return errors::Unimplemented("Not implemented by the fd filesystem");
+}
+
+Status FileDescriptorFileSystem::CreateDir(const string& d) {
+ return errors::Unimplemented("Not implemented by the fd filesystem");
+}
+
+Status FileDescriptorFileSystem::DeleteDir(const string& d) {
+ return errors::Unimplemented("Not implemented by the fd filesystem");
+}
+
+Status FileDescriptorFileSystem::RenameFile(const string& s, const string& t) {
+ return errors::Unimplemented("Not implemented by the fd filesystem");
+}
+
+Status FileDescriptorFileSystem::CanCreateTempFile(const std::string& fname,
+ bool* can_create_temp_file) {
+ *can_create_temp_file = false;
+ return OkStatus();
+}
+
+REGISTER_FILE_SYSTEM("fd", FileDescriptorFileSystem);
+
+} // namespace fcp
+} // namespace tensorflow
diff --git a/fcp/tensorflow/file_descriptor_filesystem.h b/fcp/tensorflow/file_descriptor_filesystem.h
new file mode 100644
index 0000000..86a4084
--- /dev/null
+++ b/fcp/tensorflow/file_descriptor_filesystem.h
@@ -0,0 +1,72 @@
+#ifndef FCP_TENSORFLOW_FILE_DESCRIPTOR_FILESYSTEM_H_
+#define FCP_TENSORFLOW_FILE_DESCRIPTOR_FILESYSTEM_H_
+
+#include <memory>
+#include <vector>
+
+#include "tensorflow/core/platform/file_system.h"
+
+namespace tensorflow {
+namespace fcp {
+
+// Filesystem for file descriptors that reads URIs in the form "fd:///<int>",
+// where <int> is a valid file descriptor number. This filesystem can be useful
+// to support situations in which a file descriptor is opened in Java and passed
+// to the JNI layer.
+//
+// CAVEATS:
+// * This filesystem is non-hierarchical and read-only; many functions simply
+// return tensorflow::error::code::UNIMPLEMENTED.
+// * To read a file descriptor, this filesystem makes a dup and closes the dup
+// when it is done with it. The code that originally created the URI is
+// responsible for closing the original file descriptor.
+class FileDescriptorFileSystem : public tensorflow::FileSystem {
+ public:
+ FileDescriptorFileSystem() = default;
+ ~FileDescriptorFileSystem() override = default;
+
+ tensorflow::Status NewRandomAccessFile(
+ const std::string& filename,
+ std::unique_ptr<RandomAccessFile>* result) override;
+
+ // Clears *results and stores pattern if pattern is a literal match of a valid
+ // file descriptor. As such, this does not support the full pattern matching
+ // specification as described by FileSystem::GetMatchingPaths.
+ tensorflow::Status GetMatchingPaths(
+ const std::string& pattern, std::vector<std::string>* results) override;
+
+ tensorflow::Status Stat(const std::string& fname,
+ tensorflow::FileStatistics* stats) override;
+
+ tensorflow::Status GetFileSize(const std::string& fname,
+ uint64* size) override;
+
+ // Not necessary to read TF checkpoints; these always return UNIMPLEMENTED
+ tensorflow::Status NewReadOnlyMemoryRegionFromFile(
+ const std::string& filename,
+ std::unique_ptr<ReadOnlyMemoryRegion>* result) override;
+
+ tensorflow::Status FileExists(const std::string& fname) override;
+
+ // The fd filesystem is non-hierarchical; this always returns UNIMPLEMENTED
+ tensorflow::Status GetChildren(const std::string& dir,
+ std::vector<string>* r) override;
+
+ // The fd filesystem is read-only; these always return UNIMPLEMENTED
+ tensorflow::Status NewWritableFile(
+ const std::string& fname, std::unique_ptr<WritableFile>* result) override;
+ tensorflow::Status NewAppendableFile(
+ const std::string& fname, std::unique_ptr<WritableFile>* result) override;
+ tensorflow::Status DeleteFile(const std::string& f) override;
+ tensorflow::Status CreateDir(const std::string& d) override;
+ tensorflow::Status DeleteDir(const std::string& d) override;
+ tensorflow::Status RenameFile(const std::string& s,
+ const std::string& t) override;
+ tensorflow::Status CanCreateTempFile(const std::string& fname,
+ bool* can_create_temp_file) override;
+};
+
+} // namespace fcp
+} // namespace tensorflow
+
+#endif // FCP_TENSORFLOW_FILE_DESCRIPTOR_FILESYSTEM_H_
diff --git a/fcp/tensorflow/file_descriptor_filesystem_test.cc b/fcp/tensorflow/file_descriptor_filesystem_test.cc
new file mode 100644
index 0000000..53ed024
--- /dev/null
+++ b/fcp/tensorflow/file_descriptor_filesystem_test.cc
@@ -0,0 +1,203 @@
+
+#include "fcp/tensorflow/file_descriptor_filesystem.h"
+
+#include <fcntl.h>
+
+#include <memory>
+#include <vector>
+
+#include "absl/strings/str_cat.h"
+#include "absl/strings/string_view.h"
+#include "android-base/file.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+
+namespace tensorflow {
+namespace fcp {
+namespace {
+
+using ::android::base::ReadFileToString;
+using ::testing::TempDir;
+
+static constexpr char kBadFdPath[] = "fd:///10000";
+static constexpr char kFileContent[] = "abcdefgh";
+static constexpr int kFileContentLen = 8;
+
+class FileDescriptorFileSystemTest : public ::testing::Test {
+ protected:
+ void TearDown() override {
+ if (fd_ != -1) {
+ close(fd_);
+ }
+ }
+
+ // Writes contents to a temp file and sets fd_path_ to point to it. The opened
+ // file descriptor is closed automatically in TearDown(). To be called at most
+ // once per test.
+ void CreateAndOpenFdForTest(std::string contents) {
+ ASSERT_EQ(-1, fd_); // prevent accidental double-open
+ file_name_ = TempDir() + "/fdtest";
+ android::base::WriteStringToFile(contents, file_name_);
+
+ fd_ = open(file_name_.c_str(), O_RDONLY);
+ ASSERT_NE(-1, fd_);
+
+ fd_path_ = absl::StrCat("fd:///", fd_);
+ }
+
+ void CreateTempFdForTest() {
+ ASSERT_EQ(-1, fd_); // prevent accidental double-open
+
+ file_name_ = TempDir() + "/fdtest";
+ android::base::WriteStringToFile("", file_name_);
+
+ fd_ = open(file_name_.c_str(), O_WRONLY);
+ ASSERT_NE(-1, fd_);
+
+ fd_path_ = absl::StrCat("fd:///", fd_);
+ }
+
+ FileDescriptorFileSystem fd_fs_;
+ string fd_path_;
+ string file_name_;
+
+ private:
+ int fd_ = -1;
+};
+
+TEST_F(FileDescriptorFileSystemTest, WritableFile) {
+ CreateTempFdForTest();
+
+ std::unique_ptr<WritableFile> file;
+ TF_ASSERT_OK(fd_fs_.NewWritableFile(fd_path_, &file));
+ TF_ASSERT_OK(file->Append(kFileContent));
+ TF_ASSERT_OK(file->Close());
+
+ std::string actual_content;
+ ASSERT_TRUE(ReadFileToString(file_name_, &actual_content));
+ EXPECT_EQ(kFileContent, actual_content);
+}
+
+TEST_F(FileDescriptorFileSystemTest, WritableFileFailsOnInvalidFd) {
+ std::unique_ptr<WritableFile> file;
+ EXPECT_FALSE(fd_fs_.NewWritableFile(kBadFdPath, &file).ok());
+}
+
+TEST_F(FileDescriptorFileSystemTest, MalformedPathReturnsInvalidArgument) {
+ FileStatistics stats;
+ EXPECT_EQ(fd_fs_.Stat("fd://0", &stats).code(), error::INVALID_ARGUMENT);
+ EXPECT_EQ(fd_fs_.Stat("fd://authority/0", &stats).code(),
+ error::INVALID_ARGUMENT);
+ EXPECT_EQ(fd_fs_.Stat("fd:///not-a-number", &stats).code(),
+ error::INVALID_ARGUMENT);
+}
+
+TEST_F(FileDescriptorFileSystemTest, NewRandomAccessFile) {
+ CreateAndOpenFdForTest(kFileContent);
+
+ std::unique_ptr<RandomAccessFile> file;
+ TF_ASSERT_OK(fd_fs_.NewRandomAccessFile(fd_path_, &file));
+ StringPiece content;
+ char scratch[kFileContentLen];
+ TF_ASSERT_OK(file->Read(0, kFileContentLen, &content, scratch));
+
+ EXPECT_EQ(kFileContent, content);
+}
+
+TEST_F(FileDescriptorFileSystemTest,
+ NewRandomAccessFileFailsOnRequestMoreBytes) {
+ CreateAndOpenFdForTest(kFileContent);
+
+ std::unique_ptr<RandomAccessFile> file;
+ TF_ASSERT_OK(fd_fs_.NewRandomAccessFile(fd_path_, &file));
+ StringPiece content;
+ char scratch[kFileContentLen];
+ auto status = file->Read(0, kFileContentLen + 2, &content, scratch);
+ EXPECT_EQ(status.code(), error::OUT_OF_RANGE);
+ EXPECT_EQ(status.error_message(),
+ "Read fewer bytes than requested. Total read bytes 8");
+}
+
+TEST_F(FileDescriptorFileSystemTest, NewRandomAccessFileFailsOnInvalidFd) {
+ std::unique_ptr<RandomAccessFile> file;
+ EXPECT_FALSE(fd_fs_.NewRandomAccessFile(kBadFdPath, &file).ok());
+}
+
+TEST_F(FileDescriptorFileSystemTest, NewRandomAccessFileFailsOnDirectoryFd) {
+ int dir_fd = open(TempDir().c_str(), O_RDONLY | O_DIRECTORY);
+ ASSERT_NE(-1, dir_fd);
+ string dir_fd_path = absl::StrCat("fd:///", dir_fd);
+
+ std::unique_ptr<RandomAccessFile> file;
+ EXPECT_FALSE(fd_fs_.NewRandomAccessFile(dir_fd_path, &file).ok());
+
+ close(dir_fd);
+}
+
+TEST_F(FileDescriptorFileSystemTest, GetMatchingPaths) {
+ CreateAndOpenFdForTest(kFileContent);
+
+ std::vector<string> paths;
+ TF_EXPECT_OK(fd_fs_.GetMatchingPaths(fd_path_, &paths));
+
+ ASSERT_EQ(1, paths.size());
+ EXPECT_EQ(fd_path_, paths.at(0));
+}
+
+TEST_F(FileDescriptorFileSystemTest, GetMatchingPathsReturnsEmptyOnBadFd) {
+ std::vector<string> paths;
+ TF_EXPECT_OK(fd_fs_.GetMatchingPaths(kBadFdPath, &paths));
+ EXPECT_TRUE(paths.empty());
+}
+
+TEST_F(FileDescriptorFileSystemTest, Stat) {
+ CreateAndOpenFdForTest(kFileContent);
+
+ FileStatistics stats;
+ TF_EXPECT_OK(fd_fs_.Stat(fd_path_, &stats));
+
+ EXPECT_EQ(kFileContentLen, stats.length);
+ EXPECT_GT(stats.mtime_nsec, 0);
+ EXPECT_FALSE(stats.is_directory);
+}
+
+TEST_F(FileDescriptorFileSystemTest, StatFailsOnBadFd) {
+ FileStatistics stats;
+ EXPECT_FALSE(fd_fs_.Stat(kBadFdPath, &stats).ok());
+}
+
+TEST_F(FileDescriptorFileSystemTest, GetFileSize) {
+ CreateAndOpenFdForTest(kFileContent);
+
+ uint64 size;
+ TF_EXPECT_OK(fd_fs_.GetFileSize(fd_path_, &size));
+
+ EXPECT_EQ(kFileContentLen, size);
+}
+
+TEST_F(FileDescriptorFileSystemTest, GetFileSizeFailsOnBadFd) {
+ uint64 size;
+ EXPECT_FALSE(fd_fs_.GetFileSize(kBadFdPath, &size).ok());
+}
+
+TEST_F(FileDescriptorFileSystemTest,
+ NewReadOnlyMemoryRegionFromFileReturnsUnimplemented) {
+ std::unique_ptr<ReadOnlyMemoryRegion> region;
+ EXPECT_EQ(fd_fs_.NewReadOnlyMemoryRegionFromFile(kBadFdPath, &region).code(),
+ error::UNIMPLEMENTED);
+}
+
+TEST_F(FileDescriptorFileSystemTest, FileExistsReturnsUnimplemented) {
+ EXPECT_EQ(fd_fs_.FileExists(kBadFdPath).code(), error::UNIMPLEMENTED);
+}
+
+TEST_F(FileDescriptorFileSystemTest, GetChildrenReturnsUnimplemented) {
+ std::vector<string> paths;
+ EXPECT_EQ(fd_fs_.GetChildren(kBadFdPath, &paths).code(),
+ error::UNIMPLEMENTED);
+}
+
+} // anonymous namespace
+} // namespace fcp
+} // namespace tensorflow