diff options
author | qiaoli <qiaoli@google.com> | 2023-06-27 17:26:19 +0000 |
---|---|---|
committer | qiaoli <qiaoli@google.com> | 2023-07-18 00:51:03 +0000 |
commit | 3b849307dbdfc75c68ce8918d0a5663c13fab379 (patch) | |
tree | a16bd6de61fe6a60083eefd8b3dbb191e9f2ceab | |
parent | 9e36da1f564c3b5a6c0c7565d0dc423c5e694779 (diff) | |
download | federated-compute-3b849307dbdfc75c68ce8918d0a5663c13fab379.tar.gz |
Add tensoflow filesytem support file descriptor
Bug: 281758731
Test: atest
Change-Id: I19d6d2deb8a8a66d5ef66c6a1640b867f3434b80
-rw-r--r-- | Android.bp | 14 | ||||
-rw-r--r-- | PREUPLOAD.cfg | 5 | ||||
-rw-r--r-- | fcp/client/engine/example_query_plan_engine_test.cc | 12 | ||||
-rw-r--r-- | fcp/client/engine/tflite_plan_engine_test.cc | 14 | ||||
-rw-r--r-- | fcp/client/fcp_runner.cc | 2 | ||||
-rw-r--r-- | fcp/tensorflow/file_descriptor_filesystem.cc | 250 | ||||
-rw-r--r-- | fcp/tensorflow/file_descriptor_filesystem.h | 72 | ||||
-rw-r--r-- | fcp/tensorflow/file_descriptor_filesystem_test.cc | 203 |
8 files changed, 554 insertions, 18 deletions
@@ -87,7 +87,6 @@ cc_library { }, static_libs: [ "tensorflow_core_proto_cpp_lite", - "libprotobuf-cpp-lite-ndk", ], shared_libs: [ "liblog", @@ -120,12 +119,13 @@ cc_library_static { "fcp/client/engine/tf_*.cc", "fcp/tensorflow/tf_session.cc", ], - static_libs:[ + static_libs: [ "federated-compute-cc-proto-lite", ], whole_static_libs: [ "libtflite_flex_delegate", ], + export_include_dirs: ["."], header_libs: [ "flatbuffer_headers", "libeigen", @@ -145,7 +145,6 @@ cc_library_static { "-Wno-defaulted-function-deleted", "-Wno-deprecated-declarations", ], - export_include_dirs: ["."], stl: "libc++_static", sdk_version: "current", apex_available: ["com.android.ondevicepersonalization"], @@ -156,7 +155,7 @@ filegroup { name: "fcp_native_wrapper", srcs: ["fcp/java_src/main/java/com/google/fcp/client/CallFromNativeWrapper.java"], visibility: [ - "//packages/modules/OnDevicePersonalization:__subpackages__" + "//packages/modules/OnDevicePersonalization:__subpackages__", ], } @@ -169,7 +168,7 @@ filegroup { "fcp/testdata/federation_test_select_checkpoints.pb", ], visibility: [ - "//packages/modules/OnDevicePersonalization:__subpackages__" + "//packages/modules/OnDevicePersonalization:__subpackages__", ], } @@ -177,6 +176,8 @@ cc_test { name: "fcp_plan_engine_test", srcs: [ "fcp/client/engine/tflite_plan_engine_test.cc", + "fcp/client/engine/example_query_plan_engine_test.cc", + "fcp/tensorflow/file_descriptor_filesystem_test.cc", "fcp/client/test_helpers.cc", ], test_suites: [ @@ -188,6 +189,7 @@ cc_test { "libbase_ndk", "libprotobuf-cpp-lite-ndk", "liblog", + "tensorflow_abseil", "libc++fs", // used by filesystem ], whole_static_libs: [ @@ -212,4 +214,4 @@ cc_test { "-Wno-ignored-qualifiers", "-Wno-missing-field-initializers", ], -}
\ No newline at end of file +} diff --git a/PREUPLOAD.cfg b/PREUPLOAD.cfg new file mode 100644 index 0000000..e33b85d --- /dev/null +++ b/PREUPLOAD.cfg @@ -0,0 +1,5 @@ +[Builtin Hooks] +bpfmt = true + +[Hook Scripts] +checkstyle_hook = ${REPO_ROOT}/prebuilts/checkstyle/checkstyle.py --sha ${PREUPLOAD_COMMIT}
\ No newline at end of file diff --git a/fcp/client/engine/example_query_plan_engine_test.cc b/fcp/client/engine/example_query_plan_engine_test.cc index dbad82d..638e861 100644 --- a/fcp/client/engine/example_query_plan_engine_test.cc +++ b/fcp/client/engine/example_query_plan_engine_test.cc @@ -25,8 +25,6 @@ #include <utility> #include <vector> -#include "gmock/gmock.h" -#include "gtest/gtest.h" #include "absl/container/flat_hash_map.h" #include "absl/status/status.h" #include "absl/status/statusor.h" @@ -36,7 +34,8 @@ #include "fcp/client/example_query_result.pb.h" #include "fcp/client/test_helpers.h" #include "fcp/protos/plan.pb.h" -#include "fcp/testing/testing.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" #include "tensorflow/c/checkpoint_reader.h" #include "tensorflow/c/tf_status.h" #include "tensorflow/c/tf_status_helper.h" @@ -268,7 +267,7 @@ TEST_F(ExampleQueryPlanEngineTest, PlanSucceeds) { EXPECT_THAT(result.outcome, PlanOutcome::kSuccess); auto tensors = ReadTensors(output_checkpoint_filename_); - ASSERT_OK(tensors); + // ASSERT_OK(tensors); tf::Tensor int_tensor = tensors.value()[kOutputIntTensorName]; ASSERT_EQ(int_tensor.shape(), tf::TensorShape({2})); ASSERT_EQ(int_tensor.dtype(), tf::DT_INT64); @@ -360,7 +359,8 @@ TEST_F(ExampleQueryPlanEngineTest, MultipleQueries) { EXPECT_THAT(result.outcome, PlanOutcome::kSuccess); auto tensors = ReadTensors(output_checkpoint_filename_); - ASSERT_OK(tensors); + // ASSERT_OK is not supported in AOSP. + // ASSERT_OK(tensors); tf::Tensor int_tensor = tensors.value()[kOutputIntTensorName]; ASSERT_EQ(int_tensor.shape(), tf::TensorShape({2})); ASSERT_EQ(int_tensor.dtype(), tf::DT_INT64); @@ -544,4 +544,4 @@ TEST_F(ExampleQueryPlanEngineTest, InvalidExampleQueryResultFormat) { } // anonymous namespace } // namespace engine } // namespace client -} // namespace fcp +} // namespace fcp
\ No newline at end of file diff --git a/fcp/client/engine/tflite_plan_engine_test.cc b/fcp/client/engine/tflite_plan_engine_test.cc index 8e5319e..8fffa0e 100644 --- a/fcp/client/engine/tflite_plan_engine_test.cc +++ b/fcp/client/engine/tflite_plan_engine_test.cc @@ -122,15 +122,17 @@ class TfLitePlanEngineTest : public testing::Test { FederatedComputeIORouter io_router = client_only_plan_.phase().federated_compute(); if (!io_router.input_filepath_tensor_name().empty()) { - (*inputs_)[io_router.input_filepath_tensor_name()] = - checkpoint_input_filename_; + (*inputs_)[io_router.input_filepath_tensor_name()] = checkpoint_input_fd_; } checkpoint_output_filename_ = files_impl_.CreateTempFile("output", ".ckp").value(); ASSERT_EQ(std::filesystem::file_size(checkpoint_output_filename_), 0); + int fd = open(checkpoint_output_filename_.c_str(), O_WRONLY); + ASSERT_NE(-1, fd); + checkpoint_output_fd_ = absl::StrCat("fd:///", fd); if (!io_router.output_filepath_tensor_name().empty()) { (*inputs_)[io_router.output_filepath_tensor_name()] = - checkpoint_output_filename_; + checkpoint_output_fd_; } for (const auto& tensor_spec : @@ -146,6 +148,9 @@ class TfLitePlanEngineTest : public testing::Test { client_only_plan_ = std::move(artifacts->plan); dataset_ = std::move(artifacts->dataset); checkpoint_input_filename_ = artifacts->checkpoint_filepath; + int fd = open(checkpoint_input_filename_.c_str(), O_RDONLY); + ASSERT_NE(-1, fd); + checkpoint_input_fd_ = absl::StrCat("fd:///", fd); } void ComputeDatasetStats(const std::string& collection_uri) { @@ -177,6 +182,8 @@ class TfLitePlanEngineTest : public testing::Test { Dataset dataset_; std::string checkpoint_input_filename_; std::string checkpoint_output_filename_; + std::string checkpoint_input_fd_; + std::string checkpoint_output_fd_; int num_examples_ = 0; int example_bytes_ = 0; @@ -209,7 +216,6 @@ TEST_F(TfLitePlanEngineTest, SimpleAggPlanSucceeds) { engine::PlanResult result = plan_engine.RunPlan( client_only_plan_.phase().tensorflow_spec(), client_only_plan_.tflite_graph(), std::move(inputs_), output_names_); - FCP_LOG(INFO) << "**** plan result " << result.original_status; EXPECT_THAT(result.outcome, PlanOutcome::kSuccess); EXPECT_THAT(result.output_tensors.size(), 0); diff --git a/fcp/client/fcp_runner.cc b/fcp/client/fcp_runner.cc index 4736b6a..8b645d5 100644 --- a/fcp/client/fcp_runner.cc +++ b/fcp/client/fcp_runner.cc @@ -280,8 +280,6 @@ absl::StatusOr<FLRunnerResult> RunFederatedComputation( fl_runner_result.set_contribution_result(FLRunnerResult::SUCCESS); } else { fl_runner_result.set_contribution_result(FLRunnerResult::FAIL); - std::string error_message = std::string{ - plan_result_and_checkpoint_file.plan_result.original_status.message()}; } return fl_runner_result; } diff --git a/fcp/tensorflow/file_descriptor_filesystem.cc b/fcp/tensorflow/file_descriptor_filesystem.cc new file mode 100644 index 0000000..1659839 --- /dev/null +++ b/fcp/tensorflow/file_descriptor_filesystem.cc @@ -0,0 +1,250 @@ +#include "fcp/tensorflow/file_descriptor_filesystem.h" + +#include <errno.h> +#include <sys/stat.h> +#include <unistd.h> + +#include <algorithm> +#include <memory> +#include <vector> + +#include "absl/memory/memory.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/strings/strip.h" +#include "fcp/base/monitoring.h" +#include "tensorflow/core/platform/env.h" + +namespace tensorflow { + +namespace fcp { + +using ::tensorflow::Status; + +static constexpr char kFdFilesystemPrefix[] = "fd:///"; +static constexpr size_t kMaxWriteChunkSize = 64u * 1024; // 64KB + +namespace { +// Copied from base implementation in +// //external/tensorflow/tensorflow/tsl/platform/default/posix_file_system.cc +absl::Status ReadBytesFromFd(int fd, uint64_t offset, size_t n, + absl::string_view* result, char* scratch) { + absl::Status s; + char* dst = scratch; + while (n > 0 && s.ok()) { + ssize_t r = pread(fd, dst, n, static_cast<off_t>(offset)); + if (r > 0) { + dst += r; + n -= r; + offset += r; + } else if (r == 0) { + s = absl::OutOfRangeError(absl::StrCat( + "Read fewer bytes than requested. Total read bytes ", offset)); + } else if (errno == EINTR || errno == EAGAIN) { + // Retry + } else { + s = absl::UnknownError(absl::StrCat("Failed to read: errno ", errno)); + } + } + *result = absl::string_view(scratch, dst - scratch); + return s; +} + +class FdRandomAccessFile : public RandomAccessFile { + public: + explicit FdRandomAccessFile(int fd) : fd_(fd) {} + + ~FdRandomAccessFile() override { close(fd_); } + + Status Read(uint64 offset, size_t n, StringPiece* result, + char* scratch) const override { + absl::string_view sv; + absl::Status s = ReadBytesFromFd(fd_, offset, n, &sv, scratch); + if (s.ok()) { + *result = StringPiece(sv.data(), sv.size()); + return Status(); + } else { + return Status(static_cast<tensorflow::errors::Code>(s.code()), + s.message()); + } + } + + private: + int fd_; +}; + +class FdWritableFile : public WritableFile { + public: + explicit FdWritableFile(int fd) : fd_(fd) {} + + ~FdWritableFile() override { Close(); } + + Status Append(StringPiece data) override { + return Write(data.data(), data.size()); + } + + Status Close() override { + if (has_closed_) return Status(); + close(fd_); + has_closed_ = true; + return Status(); + } + Status Flush() override { return Status(); } + Status Sync() override { return Status(); } + + private: + Status Write(const void* data, size_t data_size) { + size_t write_len = data_size; + do { + size_t chunk_size = std::min<size_t>(write_len, kMaxWriteChunkSize); + ssize_t wrote = write(fd_, data, chunk_size); + if (wrote < 0) { + return Status(tensorflow::error::Code::UNKNOWN, + absl::StrCat("Failed to write: ", errno)); + } + data = static_cast<const uint8_t*>(data) + wrote; + write_len -= wrote; + } while (write_len > 0); + return Status(); + } + int fd_; + bool has_closed_ = false; +}; + +// Gets the file descriptor in the URI. +Status GetFd(absl::string_view fname, int* result) { + // Consume scheme and empty authority (fd:///) + if (!absl::ConsumePrefix(&fname, kFdFilesystemPrefix)) { + return errors::InvalidArgument("Bad uri: ", fname); + } + + // Try to parse remainder of path as an integer fd + if (!absl::SimpleAtoi(fname, result)) { + return errors::InvalidArgument("Bad path: ", fname); + } + + return OkStatus(); +} + +} // anonymous namespace + +Status FileDescriptorFileSystem::NewRandomAccessFile( + const string& filename, std::unique_ptr<RandomAccessFile>* result) { + int fd; + TF_RETURN_IF_ERROR(GetFd(filename, &fd)); + FileStatistics stat; + TF_RETURN_IF_ERROR(Stat(filename, &stat)); // check against directory FD + + int dup_fd = dup(fd); + if (dup_fd == -1) { + return errors::Unknown("Failed to dup: errno ", errno); + } + + *result = std::make_unique<FdRandomAccessFile>(dup_fd); + return OkStatus(); +} + +Status FileDescriptorFileSystem::GetMatchingPaths( + const string& pattern, std::vector<string>* results) { + results->clear(); + FileStatistics statistics; + if (Stat(pattern, &statistics).ok()) { + results->push_back(pattern); + } + return OkStatus(); +} + +Status FileDescriptorFileSystem::Stat(const string& fname, + FileStatistics* stats) { + if (stats == nullptr) { + return errors::InvalidArgument("FileStatistics pointer must not be NULL"); + } + + int fd; + TF_RETURN_IF_ERROR(GetFd(fname, &fd)); + + struct stat st; + if (fstat(fd, &st) == -1) { + return errors::Unknown("Failed to fstat: errno ", errno); + } + + if (S_ISDIR(st.st_mode)) { + return errors::NotFound("File not found: is a directory"); + } + stats->length = st.st_size; + stats->mtime_nsec = st.st_mtime * 1e9; + stats->is_directory = S_ISDIR(st.st_mode); + + return OkStatus(); +} + +Status FileDescriptorFileSystem::GetFileSize(const string& fname, + uint64* size) { + FileStatistics stat; + TF_RETURN_IF_ERROR(Stat(fname, &stat)); + *size = stat.length; + return OkStatus(); +} + +Status FileDescriptorFileSystem::NewReadOnlyMemoryRegionFromFile( + const string& filename, std::unique_ptr<ReadOnlyMemoryRegion>* result) { + return errors::Unimplemented("Not implemented by the fd filesystem"); +} + +Status FileDescriptorFileSystem::FileExists(const string& fname) { + return errors::Unimplemented("Not implemented by the fd filesystem"); +} + +Status FileDescriptorFileSystem::GetChildren(const string& dir, + std::vector<string>* r) { + return errors::Unimplemented("fd filesystem is non-hierarchical"); +} + +Status FileDescriptorFileSystem::NewWritableFile( + const string& fname, std::unique_ptr<WritableFile>* result) { + int fd; + TF_RETURN_IF_ERROR(GetFd(fname, &fd)); + FileStatistics stat; + TF_RETURN_IF_ERROR(Stat(fname, &stat)); // check against directory FD + + int dup_fd = dup(fd); + if (dup_fd == -1) { + return errors::Unknown("Failed to dup: errno ", errno); + } + + *result = std::make_unique<FdWritableFile>(dup_fd); + return OkStatus(); +} + +Status FileDescriptorFileSystem::NewAppendableFile( + const string& fname, std::unique_ptr<WritableFile>* result) { + return errors::Unimplemented("Not implemented by the fd filesystem"); +} + +Status FileDescriptorFileSystem::DeleteFile(const string& f) { + return errors::Unimplemented("Not implemented by the fd filesystem"); +} + +Status FileDescriptorFileSystem::CreateDir(const string& d) { + return errors::Unimplemented("Not implemented by the fd filesystem"); +} + +Status FileDescriptorFileSystem::DeleteDir(const string& d) { + return errors::Unimplemented("Not implemented by the fd filesystem"); +} + +Status FileDescriptorFileSystem::RenameFile(const string& s, const string& t) { + return errors::Unimplemented("Not implemented by the fd filesystem"); +} + +Status FileDescriptorFileSystem::CanCreateTempFile(const std::string& fname, + bool* can_create_temp_file) { + *can_create_temp_file = false; + return OkStatus(); +} + +REGISTER_FILE_SYSTEM("fd", FileDescriptorFileSystem); + +} // namespace fcp +} // namespace tensorflow diff --git a/fcp/tensorflow/file_descriptor_filesystem.h b/fcp/tensorflow/file_descriptor_filesystem.h new file mode 100644 index 0000000..86a4084 --- /dev/null +++ b/fcp/tensorflow/file_descriptor_filesystem.h @@ -0,0 +1,72 @@ +#ifndef FCP_TENSORFLOW_FILE_DESCRIPTOR_FILESYSTEM_H_ +#define FCP_TENSORFLOW_FILE_DESCRIPTOR_FILESYSTEM_H_ + +#include <memory> +#include <vector> + +#include "tensorflow/core/platform/file_system.h" + +namespace tensorflow { +namespace fcp { + +// Filesystem for file descriptors that reads URIs in the form "fd:///<int>", +// where <int> is a valid file descriptor number. This filesystem can be useful +// to support situations in which a file descriptor is opened in Java and passed +// to the JNI layer. +// +// CAVEATS: +// * This filesystem is non-hierarchical and read-only; many functions simply +// return tensorflow::error::code::UNIMPLEMENTED. +// * To read a file descriptor, this filesystem makes a dup and closes the dup +// when it is done with it. The code that originally created the URI is +// responsible for closing the original file descriptor. +class FileDescriptorFileSystem : public tensorflow::FileSystem { + public: + FileDescriptorFileSystem() = default; + ~FileDescriptorFileSystem() override = default; + + tensorflow::Status NewRandomAccessFile( + const std::string& filename, + std::unique_ptr<RandomAccessFile>* result) override; + + // Clears *results and stores pattern if pattern is a literal match of a valid + // file descriptor. As such, this does not support the full pattern matching + // specification as described by FileSystem::GetMatchingPaths. + tensorflow::Status GetMatchingPaths( + const std::string& pattern, std::vector<std::string>* results) override; + + tensorflow::Status Stat(const std::string& fname, + tensorflow::FileStatistics* stats) override; + + tensorflow::Status GetFileSize(const std::string& fname, + uint64* size) override; + + // Not necessary to read TF checkpoints; these always return UNIMPLEMENTED + tensorflow::Status NewReadOnlyMemoryRegionFromFile( + const std::string& filename, + std::unique_ptr<ReadOnlyMemoryRegion>* result) override; + + tensorflow::Status FileExists(const std::string& fname) override; + + // The fd filesystem is non-hierarchical; this always returns UNIMPLEMENTED + tensorflow::Status GetChildren(const std::string& dir, + std::vector<string>* r) override; + + // The fd filesystem is read-only; these always return UNIMPLEMENTED + tensorflow::Status NewWritableFile( + const std::string& fname, std::unique_ptr<WritableFile>* result) override; + tensorflow::Status NewAppendableFile( + const std::string& fname, std::unique_ptr<WritableFile>* result) override; + tensorflow::Status DeleteFile(const std::string& f) override; + tensorflow::Status CreateDir(const std::string& d) override; + tensorflow::Status DeleteDir(const std::string& d) override; + tensorflow::Status RenameFile(const std::string& s, + const std::string& t) override; + tensorflow::Status CanCreateTempFile(const std::string& fname, + bool* can_create_temp_file) override; +}; + +} // namespace fcp +} // namespace tensorflow + +#endif // FCP_TENSORFLOW_FILE_DESCRIPTOR_FILESYSTEM_H_ diff --git a/fcp/tensorflow/file_descriptor_filesystem_test.cc b/fcp/tensorflow/file_descriptor_filesystem_test.cc new file mode 100644 index 0000000..53ed024 --- /dev/null +++ b/fcp/tensorflow/file_descriptor_filesystem_test.cc @@ -0,0 +1,203 @@ + +#include "fcp/tensorflow/file_descriptor_filesystem.h" + +#include <fcntl.h> + +#include <memory> +#include <vector> + +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "android-base/file.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "tensorflow/core/lib/core/status_test_util.h" + +namespace tensorflow { +namespace fcp { +namespace { + +using ::android::base::ReadFileToString; +using ::testing::TempDir; + +static constexpr char kBadFdPath[] = "fd:///10000"; +static constexpr char kFileContent[] = "abcdefgh"; +static constexpr int kFileContentLen = 8; + +class FileDescriptorFileSystemTest : public ::testing::Test { + protected: + void TearDown() override { + if (fd_ != -1) { + close(fd_); + } + } + + // Writes contents to a temp file and sets fd_path_ to point to it. The opened + // file descriptor is closed automatically in TearDown(). To be called at most + // once per test. + void CreateAndOpenFdForTest(std::string contents) { + ASSERT_EQ(-1, fd_); // prevent accidental double-open + file_name_ = TempDir() + "/fdtest"; + android::base::WriteStringToFile(contents, file_name_); + + fd_ = open(file_name_.c_str(), O_RDONLY); + ASSERT_NE(-1, fd_); + + fd_path_ = absl::StrCat("fd:///", fd_); + } + + void CreateTempFdForTest() { + ASSERT_EQ(-1, fd_); // prevent accidental double-open + + file_name_ = TempDir() + "/fdtest"; + android::base::WriteStringToFile("", file_name_); + + fd_ = open(file_name_.c_str(), O_WRONLY); + ASSERT_NE(-1, fd_); + + fd_path_ = absl::StrCat("fd:///", fd_); + } + + FileDescriptorFileSystem fd_fs_; + string fd_path_; + string file_name_; + + private: + int fd_ = -1; +}; + +TEST_F(FileDescriptorFileSystemTest, WritableFile) { + CreateTempFdForTest(); + + std::unique_ptr<WritableFile> file; + TF_ASSERT_OK(fd_fs_.NewWritableFile(fd_path_, &file)); + TF_ASSERT_OK(file->Append(kFileContent)); + TF_ASSERT_OK(file->Close()); + + std::string actual_content; + ASSERT_TRUE(ReadFileToString(file_name_, &actual_content)); + EXPECT_EQ(kFileContent, actual_content); +} + +TEST_F(FileDescriptorFileSystemTest, WritableFileFailsOnInvalidFd) { + std::unique_ptr<WritableFile> file; + EXPECT_FALSE(fd_fs_.NewWritableFile(kBadFdPath, &file).ok()); +} + +TEST_F(FileDescriptorFileSystemTest, MalformedPathReturnsInvalidArgument) { + FileStatistics stats; + EXPECT_EQ(fd_fs_.Stat("fd://0", &stats).code(), error::INVALID_ARGUMENT); + EXPECT_EQ(fd_fs_.Stat("fd://authority/0", &stats).code(), + error::INVALID_ARGUMENT); + EXPECT_EQ(fd_fs_.Stat("fd:///not-a-number", &stats).code(), + error::INVALID_ARGUMENT); +} + +TEST_F(FileDescriptorFileSystemTest, NewRandomAccessFile) { + CreateAndOpenFdForTest(kFileContent); + + std::unique_ptr<RandomAccessFile> file; + TF_ASSERT_OK(fd_fs_.NewRandomAccessFile(fd_path_, &file)); + StringPiece content; + char scratch[kFileContentLen]; + TF_ASSERT_OK(file->Read(0, kFileContentLen, &content, scratch)); + + EXPECT_EQ(kFileContent, content); +} + +TEST_F(FileDescriptorFileSystemTest, + NewRandomAccessFileFailsOnRequestMoreBytes) { + CreateAndOpenFdForTest(kFileContent); + + std::unique_ptr<RandomAccessFile> file; + TF_ASSERT_OK(fd_fs_.NewRandomAccessFile(fd_path_, &file)); + StringPiece content; + char scratch[kFileContentLen]; + auto status = file->Read(0, kFileContentLen + 2, &content, scratch); + EXPECT_EQ(status.code(), error::OUT_OF_RANGE); + EXPECT_EQ(status.error_message(), + "Read fewer bytes than requested. Total read bytes 8"); +} + +TEST_F(FileDescriptorFileSystemTest, NewRandomAccessFileFailsOnInvalidFd) { + std::unique_ptr<RandomAccessFile> file; + EXPECT_FALSE(fd_fs_.NewRandomAccessFile(kBadFdPath, &file).ok()); +} + +TEST_F(FileDescriptorFileSystemTest, NewRandomAccessFileFailsOnDirectoryFd) { + int dir_fd = open(TempDir().c_str(), O_RDONLY | O_DIRECTORY); + ASSERT_NE(-1, dir_fd); + string dir_fd_path = absl::StrCat("fd:///", dir_fd); + + std::unique_ptr<RandomAccessFile> file; + EXPECT_FALSE(fd_fs_.NewRandomAccessFile(dir_fd_path, &file).ok()); + + close(dir_fd); +} + +TEST_F(FileDescriptorFileSystemTest, GetMatchingPaths) { + CreateAndOpenFdForTest(kFileContent); + + std::vector<string> paths; + TF_EXPECT_OK(fd_fs_.GetMatchingPaths(fd_path_, &paths)); + + ASSERT_EQ(1, paths.size()); + EXPECT_EQ(fd_path_, paths.at(0)); +} + +TEST_F(FileDescriptorFileSystemTest, GetMatchingPathsReturnsEmptyOnBadFd) { + std::vector<string> paths; + TF_EXPECT_OK(fd_fs_.GetMatchingPaths(kBadFdPath, &paths)); + EXPECT_TRUE(paths.empty()); +} + +TEST_F(FileDescriptorFileSystemTest, Stat) { + CreateAndOpenFdForTest(kFileContent); + + FileStatistics stats; + TF_EXPECT_OK(fd_fs_.Stat(fd_path_, &stats)); + + EXPECT_EQ(kFileContentLen, stats.length); + EXPECT_GT(stats.mtime_nsec, 0); + EXPECT_FALSE(stats.is_directory); +} + +TEST_F(FileDescriptorFileSystemTest, StatFailsOnBadFd) { + FileStatistics stats; + EXPECT_FALSE(fd_fs_.Stat(kBadFdPath, &stats).ok()); +} + +TEST_F(FileDescriptorFileSystemTest, GetFileSize) { + CreateAndOpenFdForTest(kFileContent); + + uint64 size; + TF_EXPECT_OK(fd_fs_.GetFileSize(fd_path_, &size)); + + EXPECT_EQ(kFileContentLen, size); +} + +TEST_F(FileDescriptorFileSystemTest, GetFileSizeFailsOnBadFd) { + uint64 size; + EXPECT_FALSE(fd_fs_.GetFileSize(kBadFdPath, &size).ok()); +} + +TEST_F(FileDescriptorFileSystemTest, + NewReadOnlyMemoryRegionFromFileReturnsUnimplemented) { + std::unique_ptr<ReadOnlyMemoryRegion> region; + EXPECT_EQ(fd_fs_.NewReadOnlyMemoryRegionFromFile(kBadFdPath, ®ion).code(), + error::UNIMPLEMENTED); +} + +TEST_F(FileDescriptorFileSystemTest, FileExistsReturnsUnimplemented) { + EXPECT_EQ(fd_fs_.FileExists(kBadFdPath).code(), error::UNIMPLEMENTED); +} + +TEST_F(FileDescriptorFileSystemTest, GetChildrenReturnsUnimplemented) { + std::vector<string> paths; + EXPECT_EQ(fd_fs_.GetChildren(kBadFdPath, &paths).code(), + error::UNIMPLEMENTED); +} + +} // anonymous namespace +} // namespace fcp +} // namespace tensorflow |