diff options
Diffstat (limited to 'icing/index/property-existence-indexing-handler_test.cc')
| -rw-r--r-- | icing/index/property-existence-indexing-handler_test.cc | 524 |
1 files changed, 524 insertions, 0 deletions
diff --git a/icing/index/property-existence-indexing-handler_test.cc b/icing/index/property-existence-indexing-handler_test.cc new file mode 100644 index 0000000..e42fbc3 --- /dev/null +++ b/icing/index/property-existence-indexing-handler_test.cc @@ -0,0 +1,524 @@ +// Copyright (C) 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "icing/index/property-existence-indexing-handler.h" + +#include <cstdint> +#include <limits> +#include <memory> +#include <string> +#include <string_view> +#include <utility> +#include <vector> + +#include "icing/text_classifier/lib3/utils/base/status.h" +#include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "icing/absl_ports/str_cat.h" +#include "icing/document-builder.h" +#include "icing/file/filesystem.h" +#include "icing/file/portable-file-backed-proto-log.h" +#include "icing/index/hit/doc-hit-info.h" +#include "icing/index/index.h" +#include "icing/index/iterator/doc-hit-info-iterator.h" +#include "icing/legacy/index/icing-filesystem.h" +#include "icing/portable/platform.h" +#include "icing/proto/document.pb.h" +#include "icing/proto/document_wrapper.pb.h" +#include "icing/proto/schema.pb.h" +#include "icing/proto/term.pb.h" +#include "icing/schema-builder.h" +#include "icing/schema/schema-store.h" +#include "icing/schema/section.h" +#include "icing/store/document-id.h" +#include "icing/store/document-store.h" +#include "icing/testing/common-matchers.h" +#include "icing/testing/fake-clock.h" +#include "icing/testing/icu-data-file-helper.h" +#include "icing/testing/test-data.h" +#include "icing/testing/tmp-directory.h" +#include "icing/tokenization/language-segmenter-factory.h" +#include "icing/tokenization/language-segmenter.h" +#include "icing/transform/normalizer-factory.h" +#include "icing/transform/normalizer.h" +#include "icing/util/tokenized-document.h" +#include "unicode/uloc.h" + +namespace icing { +namespace lib { + +namespace { + +using ::testing::ElementsAre; +using ::testing::IsTrue; +using ::testing::Test; + +static constexpr std::string_view kTreeType = "TreeNode"; +static constexpr std::string_view kPropertyName = "name"; +static constexpr std::string_view kPropertyValue = "value"; +static constexpr std::string_view kPropertySubtrees = "subtrees"; + +static constexpr std::string_view kValueType = "Value"; +static constexpr std::string_view kPropertyBody = "body"; +static constexpr std::string_view kPropertyTimestamp = "timestamp"; +static constexpr std::string_view kPropertyScore = "score"; + +class PropertyExistenceIndexingHandlerTest : public Test { + protected: + void SetUp() override { + if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { + ICING_ASSERT_OK( + // File generated via icu_data_file rule in //icing/BUILD. + icu_data_file_helper::SetUpICUDataFile( + GetTestFilePath("icing/icu.dat"))); + } + + base_dir_ = GetTestTempDir() + "/icing_test"; + ASSERT_THAT(filesystem_.CreateDirectoryRecursively(base_dir_.c_str()), + IsTrue()); + + index_dir_ = base_dir_ + "/index"; + schema_store_dir_ = base_dir_ + "/schema_store"; + document_store_dir_ = base_dir_ + "/document_store"; + + language_segmenter_factory::SegmenterOptions segmenter_options(ULOC_US); + ICING_ASSERT_OK_AND_ASSIGN( + lang_segmenter_, + language_segmenter_factory::Create(std::move(segmenter_options))); + + ICING_ASSERT_OK_AND_ASSIGN( + normalizer_, + normalizer_factory::Create( + /*max_term_byte_size=*/std::numeric_limits<int32_t>::max())); + + ASSERT_THAT( + filesystem_.CreateDirectoryRecursively(schema_store_dir_.c_str()), + IsTrue()); + ICING_ASSERT_OK_AND_ASSIGN( + schema_store_, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaProto schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType(kTreeType) + .AddProperty(PropertyConfigBuilder() + .SetName(kPropertyName) + .SetDataTypeString(TERM_MATCH_EXACT, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName(kPropertyValue) + .SetDataTypeDocument( + kValueType, /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName(kPropertySubtrees) + .SetDataTypeDocument( + kTreeType, /*index_nested_properties=*/false) + .SetCardinality(CARDINALITY_REPEATED))) + .AddType( + SchemaTypeConfigBuilder() + .SetType(kValueType) + .AddProperty(PropertyConfigBuilder() + .SetName(kPropertyBody) + .SetDataTypeString(TERM_MATCH_EXACT, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty(PropertyConfigBuilder() + .SetName(kPropertyTimestamp) + .SetDataType(TYPE_INT64) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName(kPropertyScore) + .SetDataType(TYPE_DOUBLE) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + ICING_ASSERT_OK(schema_store_->SetSchema( + schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/true)); + + ASSERT_TRUE( + filesystem_.CreateDirectoryRecursively(document_store_dir_.c_str())); + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::CreateResult doc_store_create_result, + DocumentStore::Create(&filesystem_, document_store_dir_, &fake_clock_, + schema_store_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*namespace_id_fingerprint=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + /*initialize_stats=*/nullptr)); + document_store_ = std::move(doc_store_create_result.document_store); + } + + void TearDown() override { + document_store_.reset(); + schema_store_.reset(); + normalizer_.reset(); + lang_segmenter_.reset(); + + filesystem_.DeleteDirectoryRecursively(base_dir_.c_str()); + } + + Filesystem filesystem_; + IcingFilesystem icing_filesystem_; + FakeClock fake_clock_; + std::string base_dir_; + std::string index_dir_; + std::string schema_store_dir_; + std::string document_store_dir_; + + std::unique_ptr<LanguageSegmenter> lang_segmenter_; + std::unique_ptr<Normalizer> normalizer_; + std::unique_ptr<SchemaStore> schema_store_; + std::unique_ptr<DocumentStore> document_store_; +}; + +libtextclassifier3::StatusOr<std::unique_ptr<DocHitInfoIterator>> +QueryExistence(Index* index, std::string_view property_path) { + return index->GetIterator( + absl_ports::StrCat(kPropertyExistenceTokenPrefix, property_path), + /*term_start_index=*/0, + /*unnormalized_term_length=*/0, kSectionIdMaskAll, + TermMatchType::EXACT_ONLY, + /*need_hit_term_frequency=*/false); +} + +std::vector<DocHitInfo> GetHits(std::unique_ptr<DocHitInfoIterator> iterator) { + std::vector<DocHitInfo> infos; + while (iterator->Advance().ok()) { + infos.push_back(iterator->doc_hit_info()); + } + return infos; +} + +TEST_F(PropertyExistenceIndexingHandlerTest, HandlePropertyExistence) { + Index::Options options(index_dir_, /*index_merge_size=*/1024 * 1024, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/1024 * 8); + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<Index> index, + Index::Create(options, &filesystem_, &icing_filesystem_)); + + // Create a document with every property. + DocumentProto document0 = + DocumentBuilder() + .SetKey("icing", "uri0") + .SetSchema(std::string(kValueType)) + .AddStringProperty(std::string(kPropertyBody), "foo") + .AddInt64Property(std::string(kPropertyTimestamp), 123) + .AddDoubleProperty(std::string(kPropertyScore), 456.789) + .Build(); + // Create a document with missing body. + DocumentProto document1 = + DocumentBuilder() + .SetKey("icing", "uri1") + .SetSchema(std::string(kValueType)) + .AddInt64Property(std::string(kPropertyTimestamp), 123) + .AddDoubleProperty(std::string(kPropertyScore), 456.789) + .Build(); + // Create a document with missing timestamp. + DocumentProto document2 = + DocumentBuilder() + .SetKey("icing", "uri2") + .SetSchema(std::string(kValueType)) + .AddStringProperty(std::string(kPropertyBody), "foo") + .AddDoubleProperty(std::string(kPropertyScore), 456.789) + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN( + TokenizedDocument tokenized_document0, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + std::move(document0))); + ICING_ASSERT_OK_AND_ASSIGN( + TokenizedDocument tokenized_document1, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + std::move(document1))); + ICING_ASSERT_OK_AND_ASSIGN( + TokenizedDocument tokenized_document2, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + std::move(document2))); + ICING_ASSERT_OK_AND_ASSIGN( + DocumentId document_id0, + document_store_->Put(tokenized_document0.document())); + ICING_ASSERT_OK_AND_ASSIGN( + DocumentId document_id1, + document_store_->Put(tokenized_document1.document())); + ICING_ASSERT_OK_AND_ASSIGN( + DocumentId document_id2, + document_store_->Put(tokenized_document2.document())); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<PropertyExistenceIndexingHandler> handler, + PropertyExistenceIndexingHandler::Create(&fake_clock_, index.get())); + + // Handle all docs + EXPECT_THAT(handler->Handle(tokenized_document0, document_id0, + /*put_document_stats=*/nullptr), + IsOk()); + EXPECT_THAT(handler->Handle(tokenized_document1, document_id1, + /*put_document_stats=*/nullptr), + IsOk()); + EXPECT_THAT(handler->Handle(tokenized_document2, document_id2, + /*put_document_stats=*/nullptr), + IsOk()); + + // Get all documents that have "body". + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<DocHitInfoIterator> itr, + QueryExistence(index.get(), kPropertyBody)); + EXPECT_THAT( + GetHits(std::move(itr)), + ElementsAre(EqualsDocHitInfo(document_id2, std::vector<SectionId>{0}), + EqualsDocHitInfo(document_id0, std::vector<SectionId>{0}))); + + // Get all documents that have "timestamp". + ICING_ASSERT_OK_AND_ASSIGN(itr, + QueryExistence(index.get(), kPropertyTimestamp)); + EXPECT_THAT( + GetHits(std::move(itr)), + ElementsAre(EqualsDocHitInfo(document_id1, std::vector<SectionId>{0}), + EqualsDocHitInfo(document_id0, std::vector<SectionId>{0}))); + + // Get all documents that have "score". + ICING_ASSERT_OK_AND_ASSIGN(itr, QueryExistence(index.get(), kPropertyScore)); + EXPECT_THAT( + GetHits(std::move(itr)), + ElementsAre(EqualsDocHitInfo(document_id2, std::vector<SectionId>{0}), + EqualsDocHitInfo(document_id1, std::vector<SectionId>{0}), + EqualsDocHitInfo(document_id0, std::vector<SectionId>{0}))); +} + +TEST_F(PropertyExistenceIndexingHandlerTest, HandleNestedPropertyExistence) { + Index::Options options(index_dir_, /*index_merge_size=*/1024 * 1024, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/1024 * 8); + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<Index> index, + Index::Create(options, &filesystem_, &icing_filesystem_)); + + // Create a complex nested root_document with the following property paths. + // - name + // - subtrees + // - subtrees.name + // - subtrees.value + // - subtrees.value.timestamp + // - subtrees.subtrees + // - subtrees.subtrees.name + // - subtrees.subtrees.value + // - subtrees.subtrees.value.body + // - subtrees.subtrees.value.score + DocumentProto leaf_document = + DocumentBuilder() + .SetKey("icing", "uri") + .SetSchema(std::string(kTreeType)) + .AddStringProperty(std::string(kPropertyName), "leaf") + .AddDocumentProperty( + std::string(kPropertyValue), + DocumentBuilder() + .SetKey("icing", "uri") + .SetSchema(std::string(kValueType)) + .AddStringProperty(std::string(kPropertyBody), "foo") + .AddDoubleProperty(std::string(kPropertyScore), 456.789) + .Build()) + .Build(); + DocumentProto intermediate_document1 = + DocumentBuilder() + .SetKey("icing", "uri") + .SetSchema(std::string(kTreeType)) + .AddStringProperty(std::string(kPropertyName), "intermediate1") + .AddDocumentProperty( + std::string(kPropertyValue), + DocumentBuilder() + .SetKey("icing", "uri") + .SetSchema(std::string(kValueType)) + .AddInt64Property(std::string(kPropertyTimestamp), 123) + .Build()) + .AddDocumentProperty(std::string(kPropertySubtrees), leaf_document) + .Build(); + DocumentProto intermediate_document2 = + DocumentBuilder() + .SetKey("icing", "uri") + .SetSchema(std::string(kTreeType)) + .AddStringProperty(std::string(kPropertyName), "intermediate2") + .Build(); + DocumentProto root_document = + DocumentBuilder() + .SetKey("icing", "uri") + .SetSchema(std::string(kTreeType)) + .AddStringProperty(std::string(kPropertyName), "root") + .AddDocumentProperty(std::string(kPropertySubtrees), + intermediate_document1, intermediate_document2) + .Build(); + + // Handle root_document + ICING_ASSERT_OK_AND_ASSIGN( + TokenizedDocument tokenized_root_document, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + std::move(root_document))); + ICING_ASSERT_OK_AND_ASSIGN( + DocumentId document_id, + document_store_->Put(tokenized_root_document.document())); + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<PropertyExistenceIndexingHandler> handler, + PropertyExistenceIndexingHandler::Create(&fake_clock_, index.get())); + EXPECT_THAT(handler->Handle(tokenized_root_document, document_id, + /*put_document_stats=*/nullptr), + IsOk()); + + // Check that the above property paths can be found by query. + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<DocHitInfoIterator> itr, + QueryExistence(index.get(), "name")); + EXPECT_THAT( + GetHits(std::move(itr)), + ElementsAre(EqualsDocHitInfo(document_id, std::vector<SectionId>{0}))); + + ICING_ASSERT_OK_AND_ASSIGN(itr, QueryExistence(index.get(), "subtrees")); + EXPECT_THAT( + GetHits(std::move(itr)), + ElementsAre(EqualsDocHitInfo(document_id, std::vector<SectionId>{0}))); + + ICING_ASSERT_OK_AND_ASSIGN(itr, QueryExistence(index.get(), "subtrees.name")); + EXPECT_THAT( + GetHits(std::move(itr)), + ElementsAre(EqualsDocHitInfo(document_id, std::vector<SectionId>{0}))); + + ICING_ASSERT_OK_AND_ASSIGN(itr, + QueryExistence(index.get(), "subtrees.value")); + EXPECT_THAT( + GetHits(std::move(itr)), + ElementsAre(EqualsDocHitInfo(document_id, std::vector<SectionId>{0}))); + + ICING_ASSERT_OK_AND_ASSIGN( + itr, QueryExistence(index.get(), "subtrees.value.timestamp")); + EXPECT_THAT( + GetHits(std::move(itr)), + ElementsAre(EqualsDocHitInfo(document_id, std::vector<SectionId>{0}))); + + ICING_ASSERT_OK_AND_ASSIGN(itr, + QueryExistence(index.get(), "subtrees.subtrees")); + EXPECT_THAT( + GetHits(std::move(itr)), + ElementsAre(EqualsDocHitInfo(document_id, std::vector<SectionId>{0}))); + + ICING_ASSERT_OK_AND_ASSIGN( + itr, QueryExistence(index.get(), "subtrees.subtrees.name")); + EXPECT_THAT( + GetHits(std::move(itr)), + ElementsAre(EqualsDocHitInfo(document_id, std::vector<SectionId>{0}))); + + ICING_ASSERT_OK_AND_ASSIGN( + itr, QueryExistence(index.get(), "subtrees.subtrees.value")); + EXPECT_THAT( + GetHits(std::move(itr)), + ElementsAre(EqualsDocHitInfo(document_id, std::vector<SectionId>{0}))); + + ICING_ASSERT_OK_AND_ASSIGN( + itr, QueryExistence(index.get(), "subtrees.subtrees.value.body")); + EXPECT_THAT( + GetHits(std::move(itr)), + ElementsAre(EqualsDocHitInfo(document_id, std::vector<SectionId>{0}))); + + ICING_ASSERT_OK_AND_ASSIGN( + itr, QueryExistence(index.get(), "subtrees.subtrees.value.score")); + EXPECT_THAT( + GetHits(std::move(itr)), + ElementsAre(EqualsDocHitInfo(document_id, std::vector<SectionId>{0}))); +} + +TEST_F(PropertyExistenceIndexingHandlerTest, SingleEmptyStringIsNonExisting) { + Index::Options options(index_dir_, /*index_merge_size=*/1024 * 1024, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/1024 * 8); + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<Index> index, + Index::Create(options, &filesystem_, &icing_filesystem_)); + + // Create a document with one empty body. + DocumentProto document0 = + DocumentBuilder() + .SetKey("icing", "uri0") + .SetSchema(std::string(kValueType)) + .AddStringProperty(std::string(kPropertyBody), "") + .Build(); + // Create a document with two empty body. + DocumentProto document1 = + DocumentBuilder() + .SetKey("icing", "uri1") + .SetSchema(std::string(kValueType)) + .AddStringProperty(std::string(kPropertyBody), "", "") + .Build(); + // Create a document with one non-empty body. + DocumentProto document2 = + DocumentBuilder() + .SetKey("icing", "uri2") + .SetSchema(std::string(kValueType)) + .AddStringProperty(std::string(kPropertyBody), "foo") + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN( + TokenizedDocument tokenized_document0, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + std::move(document0))); + ICING_ASSERT_OK_AND_ASSIGN( + TokenizedDocument tokenized_document1, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + std::move(document1))); + ICING_ASSERT_OK_AND_ASSIGN( + TokenizedDocument tokenized_document2, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + std::move(document2))); + ICING_ASSERT_OK_AND_ASSIGN( + DocumentId document_id0, + document_store_->Put(tokenized_document0.document())); + ICING_ASSERT_OK_AND_ASSIGN( + DocumentId document_id1, + document_store_->Put(tokenized_document1.document())); + ICING_ASSERT_OK_AND_ASSIGN( + DocumentId document_id2, + document_store_->Put(tokenized_document2.document())); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<PropertyExistenceIndexingHandler> handler, + PropertyExistenceIndexingHandler::Create(&fake_clock_, index.get())); + + // Handle all docs + EXPECT_THAT(handler->Handle(tokenized_document0, document_id0, + /*put_document_stats=*/nullptr), + IsOk()); + EXPECT_THAT(handler->Handle(tokenized_document1, document_id1, + /*put_document_stats=*/nullptr), + IsOk()); + EXPECT_THAT(handler->Handle(tokenized_document2, document_id2, + /*put_document_stats=*/nullptr), + IsOk()); + + // Check that the documents that have one or two empty bodies will not be + // considered as having a body property. + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<DocHitInfoIterator> itr, + QueryExistence(index.get(), kPropertyBody)); + EXPECT_THAT( + GetHits(std::move(itr)), + ElementsAre(EqualsDocHitInfo(document_id2, std::vector<SectionId>{0}))); +} + +} // namespace + +} // namespace lib +} // namespace icing |