From b0edd034dee25db7a3e42243b95c66eaf95e8a81 Mon Sep 17 00:00:00 2001 From: "sanjay@google.com" Date: Tue, 16 Oct 2012 23:22:36 +0000 Subject: Release 1.7 contains small fixes. Details: * Fix shared library building. * Reorganize linking commands so flags like --as-needed can be passed. * C binding exports version numbers. * Fix small typos in documention. git-svn-id: http://leveldb.googlecode.com/svn/trunk@69 62dab493-f737-651d-591e-8d6aee1b9529 --- Makefile | 52 +++++++++++++++++++++++++-------------------------- build_detect_platform | 20 +++++++++++++------- db/c.cc | 10 ++++++++++ db/c_test.c | 3 +++ doc/index.html | 6 +++--- doc/log_format.txt | 4 ++-- doc/table_format.txt | 12 +++++++----- include/leveldb/c.h | 6 ++++++ include/leveldb/db.h | 2 +- 9 files changed, 70 insertions(+), 45 deletions(-) diff --git a/Makefile b/Makefile index 4dd1366..fdc66f9 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,6 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. See the AUTHORS file for names of contributors. -# Inherit some settings from environment variables, if available -INSTALL_PATH ?= $(CURDIR) - #----------------------------------------------- # Uncomment exactly one of the lines labelled (A), (B), and (C) below # to switch between compilation modes. @@ -24,6 +21,7 @@ CFLAGS += -I. -I./include $(PLATFORM_CCFLAGS) $(OPT) CXXFLAGS += -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) LDFLAGS += $(PLATFORM_LDFLAGS) +LIBS += $(PLATFORM_LIBS) LIBOBJECTS = $(SOURCES:.cc=.o) MEMENVOBJECTS = $(MEMENV_SOURCES:.cc=.o) @@ -71,7 +69,7 @@ SHARED = $(SHARED1) else # Update db.h if you change these. SHARED_MAJOR = 1 -SHARED_MINOR = 6 +SHARED_MINOR = 7 SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT) SHARED2 = $(SHARED1).$(SHARED_MAJOR) SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR) @@ -83,7 +81,7 @@ $(SHARED2): $(SHARED3) endif $(SHARED3): - $(CXX) $(SOURCES) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(INSTALL_PATH)$(SHARED2) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -o $(SHARED3) + $(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED2) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SOURCES) -o $(SHARED3) $(LIBS) endif # PLATFORM_SHARED_EXT @@ -101,74 +99,74 @@ $(LIBRARY): $(LIBOBJECTS) $(AR) -rs $@ $(LIBOBJECTS) db_bench: db/db_bench.o $(LIBOBJECTS) $(TESTUTIL) - $(CXX) db/db_bench.o $(LIBOBJECTS) $(TESTUTIL) -o $@ $(LDFLAGS) + $(CXX) $(LDFLAGS) db/db_bench.o $(LIBOBJECTS) $(TESTUTIL) -o $@ $(LIBS) db_bench_sqlite3: doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL) - $(CXX) doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL) -o $@ $(LDFLAGS) -lsqlite3 + $(CXX) $(LDFLAGS) doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL) -o $@ -lsqlite3 $(LIBS) db_bench_tree_db: doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) - $(CXX) doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) -o $@ $(LDFLAGS) -lkyotocabinet + $(CXX) $(LDFLAGS) doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) -o $@ -lkyotocabinet $(LIBS) arena_test: util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS) + $(CXX) $(LDFLAGS) util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) bloom_test: util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS) + $(CXX) $(LDFLAGS) util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) c_test: db/c_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) db/c_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS) + $(CXX) $(LDFLAGS) db/c_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) cache_test: util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS) + $(CXX) $(LDFLAGS) util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) coding_test: util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS) + $(CXX) $(LDFLAGS) util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) corruption_test: db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS) + $(CXX) $(LDFLAGS) db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) crc32c_test: util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS) + $(CXX) $(LDFLAGS) util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) db_test: db/db_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) db/db_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS) + $(CXX) $(LDFLAGS) db/db_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) dbformat_test: db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS) + $(CXX) $(LDFLAGS) db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) env_test: util/env_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) util/env_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS) + $(CXX) $(LDFLAGS) util/env_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) filename_test: db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS) + $(CXX) $(LDFLAGS) db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) filter_block_test: table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS) + $(CXX) $(LDFLAGS) table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) log_test: db/log_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) db/log_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS) + $(CXX) $(LDFLAGS) db/log_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) table_test: table/table_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) table/table_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS) + $(CXX) $(LDFLAGS) table/table_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) skiplist_test: db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS) + $(CXX) $(LDFLAGS) db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) version_edit_test: db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS) + $(CXX) $(LDFLAGS) db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) version_set_test: db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS) + $(CXX) $(LDFLAGS) db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) write_batch_test: db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS) - $(CXX) db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS) + $(CXX) $(LDFLAGS) db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(MEMENVLIBRARY) : $(MEMENVOBJECTS) rm -f $@ $(AR) -rs $@ $(MEMENVOBJECTS) memenv_test : helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) - $(CXX) helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) -o $@ $(LDFLAGS) + $(CXX) $(LDFLAGS) helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) -o $@ $(LIBS) ifeq ($(PLATFORM), IOS) # For iOS, create universal object files to be used on both the simulator and diff --git a/build_detect_platform b/build_detect_platform index 83bbe42..7cdb793 100644 --- a/build_detect_platform +++ b/build_detect_platform @@ -7,8 +7,11 @@ # CC C Compiler path # CXX C++ Compiler path # PLATFORM_LDFLAGS Linker flags +# PLATFORM_LIBS Libraries flags # PLATFORM_SHARED_EXT Extension for shared libraries # PLATFORM_SHARED_LDFLAGS Flags for building shared library +# This flag is embedded just before the name +# of the shared library without intervening spaces # PLATFORM_SHARED_CFLAGS Flags for compiling objects for shared library # PLATFORM_CCFLAGS C compiler flags # PLATFORM_CXXFLAGS C++ compiler flags. Will contain: @@ -51,6 +54,7 @@ CROSS_COMPILE= PLATFORM_CCFLAGS= PLATFORM_CXXFLAGS= PLATFORM_LDFLAGS= +PLATFORM_LIBS= PLATFORM_SHARED_EXT="so" PLATFORM_SHARED_LDFLAGS="-shared -Wl,-soname -Wl," PLATFORM_SHARED_CFLAGS="-fPIC" @@ -68,7 +72,8 @@ case "$TARGET_OS" in PLATFORM=OS_MACOSX COMMON_FLAGS="$MEMCMP_FLAG -DOS_MACOSX" PLATFORM_SHARED_EXT=dylib - PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name " + [ -z "$INSTALL_PATH" ] && INSTALL_PATH=`pwd` + PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name $INSTALL_PATH/" PORT_FILE=port/port_posix.cc ;; Linux) @@ -80,19 +85,19 @@ case "$TARGET_OS" in SunOS) PLATFORM=OS_SOLARIS COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_SOLARIS" - PLATFORM_LDFLAGS="-lpthread -lrt" + PLATFORM_LIBS="-lpthread -lrt" PORT_FILE=port/port_posix.cc ;; FreeBSD) PLATFORM=OS_FREEBSD COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_FREEBSD" - PLATFORM_LDFLAGS="-lpthread" + PLATFORM_LIBS="-lpthread" PORT_FILE=port/port_posix.cc ;; NetBSD) PLATFORM=OS_NETBSD COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_NETBSD" - PLATFORM_LDFLAGS="-lpthread -lgcc_s" + PLATFORM_LIBS="-lpthread -lgcc_s" PORT_FILE=port/port_posix.cc ;; OpenBSD) @@ -104,7 +109,7 @@ case "$TARGET_OS" in DragonFly) PLATFORM=OS_DRAGONFLYBSD COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_DRAGONFLYBSD" - PLATFORM_LDFLAGS="-lpthread" + PLATFORM_LIBS="-lpthread" PORT_FILE=port/port_posix.cc ;; OS_ANDROID_CROSSCOMPILE) @@ -169,7 +174,7 @@ EOF EOF if [ "$?" = 0 ]; then COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY" - PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lsnappy" + PLATFORM_LIBS="$PLATFORM_LIBS -lsnappy" fi # Test whether tcmalloc is available @@ -177,7 +182,7 @@ EOF int main() {} EOF if [ "$?" = 0 ]; then - PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltcmalloc" + PLATFORM_LIBS="$PLATFORM_LIBS -ltcmalloc" fi fi @@ -188,6 +193,7 @@ echo "CC=$CC" >> $OUTPUT echo "CXX=$CXX" >> $OUTPUT echo "PLATFORM=$PLATFORM" >> $OUTPUT echo "PLATFORM_LDFLAGS=$PLATFORM_LDFLAGS" >> $OUTPUT +echo "PLATFORM_LIBS=$PLATFORM_LIBS" >> $OUTPUT echo "PLATFORM_CCFLAGS=$PLATFORM_CCFLAGS" >> $OUTPUT echo "PLATFORM_CXXFLAGS=$PLATFORM_CXXFLAGS" >> $OUTPUT echo "PLATFORM_SHARED_CFLAGS=$PLATFORM_SHARED_CFLAGS" >> $OUTPUT diff --git a/db/c.cc b/db/c.cc index ed5c4cb..08ff0ad 100644 --- a/db/c.cc +++ b/db/c.cc @@ -24,6 +24,8 @@ using leveldb::Env; using leveldb::FileLock; using leveldb::FilterPolicy; using leveldb::Iterator; +using leveldb::kMajorVersion; +using leveldb::kMinorVersion; using leveldb::Logger; using leveldb::NewBloomFilterPolicy; using leveldb::NewLRUCache; @@ -582,4 +584,12 @@ void leveldb_free(void* ptr) { free(ptr); } +int leveldb_major_version() { + return kMajorVersion; +} + +int leveldb_minor_version() { + return kMinorVersion; +} + } // end extern "C" diff --git a/db/c_test.c b/db/c_test.c index 9d6dfec..7cd5ee0 100644 --- a/db/c_test.c +++ b/db/c_test.c @@ -165,6 +165,9 @@ int main(int argc, char** argv) { char* err = NULL; int run = -1; + CheckCondition(leveldb_major_version() >= 1); + CheckCondition(leveldb_minor_version() >= 1); + snprintf(dbname, sizeof(dbname), "%s/leveldb_c_test-%d", GetTempDir(), diff --git a/doc/index.html b/doc/index.html index cd29341..3ed0ed9 100644 --- a/doc/index.html +++ b/doc/index.html @@ -420,7 +420,7 @@ The preceding code associates a based filtering policy with the database. Bloom filter based filtering relies on keeping some number of bits of data in memory per key (in this case 10 bits per key since that is the argument we passed -to NewBloomFilter). This filter will reduce the number of unnecessary +to NewBloomFilterPolicy). This filter will reduce the number of unnecessary disk reads needed for Get() calls by a factor of approximately a 100. Increasing the bits per key will lead to a larger reduction at the cost of more memory usage. We recommend that @@ -430,7 +430,7 @@ lot of random reads set a filter policy. If you are using a custom comparator, you should ensure that the filter policy you are using is compatible with your comparator. For example, consider a comparator that ignores trailing spaces when comparing keys. -NewBloomFilter must not be used with such a comparator. +NewBloomFilterPolicy must not be used with such a comparator. Instead, the application should provide a custom filter policy that also ignores trailing spaces. For example:
@@ -438,7 +438,7 @@ also ignores trailing spaces.  For example:
    private:
     FilterPolicy* builtin_policy_;
    public:
-    CustomFilterPolicy() : builtin_policy_(NewBloomFilter(10)) { }
+    CustomFilterPolicy() : builtin_policy_(NewBloomFilterPolicy(10)) { }
     ~CustomFilterPolicy() { delete builtin_policy_; }
 
     const char* Name() const { return "IgnoreTrailingSpacesFilter"; }
diff --git a/doc/log_format.txt b/doc/log_format.txt
index 3a0414b..5228f62 100644
--- a/doc/log_format.txt
+++ b/doc/log_format.txt
@@ -4,8 +4,8 @@ exception is that the tail of the file may contain a partial block.
 Each block consists of a sequence of records:
    block := record* trailer?
    record :=
-	checksum: uint32	// crc32c of type and data[]
-	length: uint16
+	checksum: uint32	// crc32c of type and data[] ; little-endian
+	length: uint16		// little-endian
 	type: uint8		// One of FULL, FIRST, MIDDLE, LAST
 	data: uint8[length]
 
diff --git a/doc/table_format.txt b/doc/table_format.txt
index d0f3065..ca8f9b4 100644
--- a/doc/table_format.txt
+++ b/doc/table_format.txt
@@ -18,6 +18,8 @@ The file contains internal pointers.  Each such pointer is called
 a BlockHandle and contains the following information:
   offset:	    varint64
   size:		    varint64
+See https://developers.google.com/protocol-buffers/docs/encoding#varints
+for an explanation of varint64 format.
 
 (1) The sequence of key/value pairs in the file are stored in sorted
 order and partitioned into a sequence of data blocks.  These blocks
@@ -41,11 +43,11 @@ BlockHandle for the data block.
 
 (6) At the very end of the file is a fixed length footer that contains
 the BlockHandle of the metaindex and index blocks as well as a magic number.
-       metaindex_handle:       char[p];    // Block handle for metaindex
-       index_handle:	       char[q];    // Block handle for index
-       padding:		       char[40-p-q]; // 0 bytes to make fixed length
-       			 	       // (40==2*BlockHandle::kMaxEncodedLength)
-       magic:		       fixed64;    // == 0xdb4775248b80fb57
+       metaindex_handle: char[p];    // Block handle for metaindex
+       index_handle:     char[q];    // Block handle for index
+       padding:          char[40-p-q]; // zeroed bytes to make fixed length
+                                       // (40==2*BlockHandle::kMaxEncodedLength)
+       magic:            fixed64;    // == 0xdb4775248b80fb57 (little-endian)
 
 "filter" Meta Block
 -------------------
diff --git a/include/leveldb/c.h b/include/leveldb/c.h
index 9cee971..1fa5886 100644
--- a/include/leveldb/c.h
+++ b/include/leveldb/c.h
@@ -278,6 +278,12 @@ extern void leveldb_env_destroy(leveldb_env_t*);
    malloc()-ed memory returned by this library. */
 extern void leveldb_free(void* ptr);
 
+/* Return the major version number for this release. */
+extern int leveldb_major_version();
+
+/* Return the minor version number for this release. */
+extern int leveldb_minor_version();
+
 #ifdef __cplusplus
 }  /* end extern "C" */
 #endif
diff --git a/include/leveldb/db.h b/include/leveldb/db.h
index a50ac69..79142f5 100644
--- a/include/leveldb/db.h
+++ b/include/leveldb/db.h
@@ -14,7 +14,7 @@ namespace leveldb {
 
 // Update Makefile if you change these
 static const int kMajorVersion = 1;
-static const int kMinorVersion = 6;
+static const int kMinorVersion = 7;
 
 struct Options;
 struct ReadOptions;
-- 
cgit v1.2.3


From a842f7ed2aff4593fb6d455fedc3d6724868be5a Mon Sep 17 00:00:00 2001
From: "sanjay@google.com"
 
Date: Thu, 27 Dec 2012 18:43:59 +0000
Subject: added utility to dump leveldb files

git-svn-id: http://leveldb.googlecode.com/svn/trunk@70 62dab493-f737-651d-591e-8d6aee1b9529
---
 Makefile              |   7 +-
 build_detect_platform |   3 +-
 db/leveldb_main.cc    | 238 ++++++++++++++++++++++++++++++++++++++++++++++++++
 include/leveldb/db.h  |   2 +-
 port/port_posix.h     |   8 +-
 5 files changed, 253 insertions(+), 5 deletions(-)
 create mode 100644 db/leveldb_main.cc

diff --git a/Makefile b/Makefile
index fdc66f9..fef085b 100644
--- a/Makefile
+++ b/Makefile
@@ -50,7 +50,7 @@ TESTS = \
 	version_set_test \
 	write_batch_test
 
-PROGRAMS = db_bench $(TESTS)
+PROGRAMS = db_bench leveldbutil $(TESTS)
 BENCHMARKS = db_bench_sqlite3 db_bench_tree_db
 
 LIBRARY = libleveldb.a
@@ -69,7 +69,7 @@ SHARED = $(SHARED1)
 else
 # Update db.h if you change these.
 SHARED_MAJOR = 1
-SHARED_MINOR = 7
+SHARED_MINOR = 8
 SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT)
 SHARED2 = $(SHARED1).$(SHARED_MAJOR)
 SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR)
@@ -107,6 +107,9 @@ db_bench_sqlite3: doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL)
 db_bench_tree_db: doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL)
 	$(CXX) $(LDFLAGS) doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) -o $@ -lkyotocabinet $(LIBS)
 
+leveldbutil: db/leveldb_main.o $(LIBOBJECTS)
+	$(CXX) $(LDFLAGS) db/leveldb_main.o $(LIBOBJECTS) -o $@ $(LIBS)
+
 arena_test: util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) $(LDFLAGS) util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
 
diff --git a/build_detect_platform b/build_detect_platform
index 7cdb793..5801d20 100644
--- a/build_detect_platform
+++ b/build_detect_platform
@@ -141,7 +141,8 @@ DIRS="$PREFIX/db $PREFIX/util $PREFIX/table"
 set -f # temporarily disable globbing so that our patterns aren't expanded
 PRUNE_TEST="-name *test*.cc -prune"
 PRUNE_BENCH="-name *_bench.cc -prune"
-PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -print | sort | sed "s,^$PREFIX/,," | tr "\n" " "`
+PRUNE_TOOL="-name leveldb_main.cc -prune"
+PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o $PRUNE_TOOL -o -name '*.cc' -print | sort | sed "s,^$PREFIX/,," | tr "\n" " "`
 
 set +f # re-enable globbing
 
diff --git a/db/leveldb_main.cc b/db/leveldb_main.cc
new file mode 100644
index 0000000..995d761
--- /dev/null
+++ b/db/leveldb_main.cc
@@ -0,0 +1,238 @@
+// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include 
+#include "db/dbformat.h"
+#include "db/filename.h"
+#include "db/log_reader.h"
+#include "db/version_edit.h"
+#include "db/write_batch_internal.h"
+#include "leveldb/env.h"
+#include "leveldb/iterator.h"
+#include "leveldb/options.h"
+#include "leveldb/status.h"
+#include "leveldb/table.h"
+#include "leveldb/write_batch.h"
+#include "util/logging.h"
+
+namespace leveldb {
+
+namespace {
+
+bool GuessType(const std::string& fname, FileType* type) {
+  size_t pos = fname.rfind('/');
+  std::string basename;
+  if (pos == std::string::npos) {
+    basename = fname;
+  } else {
+    basename = std::string(fname.data() + pos + 1, fname.size() - pos - 1);
+  }
+  uint64_t ignored;
+  return ParseFileName(basename, &ignored, type);
+}
+
+// Notified when log reader encounters corruption.
+class CorruptionReporter : public log::Reader::Reporter {
+ public:
+  virtual void Corruption(size_t bytes, const Status& status) {
+    printf("corruption: %d bytes; %s\n",
+            static_cast(bytes),
+            status.ToString().c_str());
+  }
+};
+
+// Print contents of a log file. (*func)() is called on every record.
+bool PrintLogContents(Env* env, const std::string& fname,
+                      void (*func)(Slice)) {
+  SequentialFile* file;
+  Status s = env->NewSequentialFile(fname, &file);
+  if (!s.ok()) {
+    fprintf(stderr, "%s\n", s.ToString().c_str());
+    return false;
+  }
+  CorruptionReporter reporter;
+  log::Reader reader(file, &reporter, true, 0);
+  Slice record;
+  std::string scratch;
+  while (reader.ReadRecord(&record, &scratch)) {
+    printf("--- offset %llu; ",
+           static_cast(reader.LastRecordOffset()));
+    (*func)(record);
+  }
+  delete file;
+  return true;
+}
+
+// Called on every item found in a WriteBatch.
+class WriteBatchItemPrinter : public WriteBatch::Handler {
+ public:
+  uint64_t offset_;
+  uint64_t sequence_;
+
+  virtual void Put(const Slice& key, const Slice& value) {
+    printf("  put '%s' '%s'\n",
+           EscapeString(key).c_str(),
+           EscapeString(value).c_str());
+  }
+  virtual void Delete(const Slice& key) {
+    printf("  del '%s'\n",
+           EscapeString(key).c_str());
+  }
+};
+
+
+// Called on every log record (each one of which is a WriteBatch)
+// found in a kLogFile.
+static void WriteBatchPrinter(Slice record) {
+  if (record.size() < 12) {
+    printf("log record length %d is too small\n",
+           static_cast(record.size()));
+    return;
+  }
+  WriteBatch batch;
+  WriteBatchInternal::SetContents(&batch, record);
+  printf("sequence %llu\n",
+         static_cast(WriteBatchInternal::Sequence(&batch)));
+  WriteBatchItemPrinter batch_item_printer;
+  Status s = batch.Iterate(&batch_item_printer);
+  if (!s.ok()) {
+    printf("  error: %s\n", s.ToString().c_str());
+  }
+}
+
+bool DumpLog(Env* env, const std::string& fname) {
+  return PrintLogContents(env, fname, WriteBatchPrinter);
+}
+
+// Called on every log record (each one of which is a WriteBatch)
+// found in a kDescriptorFile.
+static void VersionEditPrinter(Slice record) {
+  VersionEdit edit;
+  Status s = edit.DecodeFrom(record);
+  if (!s.ok()) {
+    printf("%s\n", s.ToString().c_str());
+    return;
+  }
+  printf("%s", edit.DebugString().c_str());
+}
+
+bool DumpDescriptor(Env* env, const std::string& fname) {
+  return PrintLogContents(env, fname, VersionEditPrinter);
+}
+
+bool DumpTable(Env* env, const std::string& fname) {
+  uint64_t file_size;
+  RandomAccessFile* file = NULL;
+  Table* table = NULL;
+  Status s = env->GetFileSize(fname, &file_size);
+  if (s.ok()) {
+    s = env->NewRandomAccessFile(fname, &file);
+  }
+  if (s.ok()) {
+    // We use the default comparator, which may or may not match the
+    // comparator used in this database. However this should not cause
+    // problems since we only use Table operations that do not require
+    // any comparisons.  In particular, we do not call Seek or Prev.
+    s = Table::Open(Options(), file, file_size, &table);
+  }
+  if (!s.ok()) {
+    fprintf(stderr, "%s\n", s.ToString().c_str());
+    delete table;
+    delete file;
+    return false;
+  }
+
+  ReadOptions ro;
+  ro.fill_cache = false;
+  Iterator* iter = table->NewIterator(ro);
+  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+    ParsedInternalKey key;
+    if (!ParseInternalKey(iter->key(), &key)) {
+      printf("badkey '%s' => '%s'\n",
+             EscapeString(iter->key()).c_str(),
+             EscapeString(iter->value()).c_str());
+    } else {
+      char kbuf[20];
+      const char* type;
+      if (key.type == kTypeDeletion) {
+        type = "del";
+      } else if (key.type == kTypeValue) {
+        type = "val";
+      } else {
+        snprintf(kbuf, sizeof(kbuf), "%d", static_cast(key.type));
+        type = kbuf;
+      }
+      printf("'%s' @ %8llu : %s => '%s'\n",
+             EscapeString(key.user_key).c_str(),
+             static_cast(key.sequence),
+             type,
+             EscapeString(iter->value()).c_str());
+    }
+  }
+  s = iter->status();
+  if (!s.ok()) {
+    printf("iterator error: %s\n", s.ToString().c_str());
+  }
+
+  delete iter;
+  delete table;
+  delete file;
+  return true;
+}
+
+bool DumpFile(Env* env, const std::string& fname) {
+  FileType ftype;
+  if (!GuessType(fname, &ftype)) {
+    fprintf(stderr, "%s: unknown file type\n", fname.c_str());
+    return false;
+  }
+  switch (ftype) {
+    case kLogFile:         return DumpLog(env, fname);
+    case kDescriptorFile:  return DumpDescriptor(env, fname);
+    case kTableFile:       return DumpTable(env, fname);
+
+    default: {
+      fprintf(stderr, "%s: not a dump-able file type\n", fname.c_str());
+      break;
+    }
+  }
+  return false;
+}
+
+bool HandleDumpCommand(Env* env, char** files, int num) {
+  bool ok = true;
+  for (int i = 0; i < num; i++) {
+    ok &= DumpFile(env, files[i]);
+  }
+  return ok;
+}
+
+}
+}  // namespace leveldb
+
+static void Usage() {
+  fprintf(
+      stderr,
+      "Usage: leveldbutil command...\n"
+      "   dump files...         -- dump contents of specified files\n"
+      );
+}
+
+int main(int argc, char** argv) {
+  leveldb::Env* env = leveldb::Env::Default();
+  bool ok = true;
+  if (argc < 2) {
+    Usage();
+    ok = false;
+  } else {
+    std::string command = argv[1];
+    if (command == "dump") {
+      ok = leveldb::HandleDumpCommand(env, argv+2, argc-2);
+    } else {
+      Usage();
+      ok = false;
+    }
+  }
+  return (ok ? 0 : 1);
+}
diff --git a/include/leveldb/db.h b/include/leveldb/db.h
index 79142f5..f1e70a0 100644
--- a/include/leveldb/db.h
+++ b/include/leveldb/db.h
@@ -14,7 +14,7 @@ namespace leveldb {
 
 // Update Makefile if you change these
 static const int kMajorVersion = 1;
-static const int kMinorVersion = 7;
+static const int kMinorVersion = 8;
 
 struct Options;
 struct ReadOptions;
diff --git a/port/port_posix.h b/port/port_posix.h
index 6ca352e..f2b89bf 100644
--- a/port/port_posix.h
+++ b/port/port_posix.h
@@ -26,11 +26,17 @@
   #include 
   #define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN)
 #elif defined(OS_OPENBSD) || defined(OS_NETBSD) ||\
-      defined(OS_DRAGONFLYBSD) || defined(OS_ANDROID)
+      defined(OS_DRAGONFLYBSD)
   #include 
   #include 
 #elif defined(OS_HPUX)
   #define PLATFORM_IS_LITTLE_ENDIAN false
+#elif defined(OS_ANDROID)
+  // Due to a bug in the NDK x86  definition,
+  // _BYTE_ORDER must be used instead of __BYTE_ORDER on Android.
+  // See http://code.google.com/p/android/issues/detail?id=39824
+  #include 
+  #define PLATFORM_IS_LITTLE_ENDIAN  (_BYTE_ORDER == _LITTLE_ENDIAN)
 #else
   #include 
 #endif
-- 
cgit v1.2.3


From 85e27d04530c0323957f28c3f47a7d9f6efeceb0 Mon Sep 17 00:00:00 2001
From: "dgrogan@chromium.org"
 
Date: Mon, 7 Jan 2013 21:28:18 +0000
Subject: Fix corruption bug found and analyzed by dhruba@gmail.com

https://groups.google.com/d/msg/leveldb/Kc9JxuIUu5A/9P0N9RL4ar8J


git-svn-id: http://leveldb.googlecode.com/svn/trunk@71 62dab493-f737-651d-591e-8d6aee1b9529
---
 Makefile             |  2 +-
 db/db_test.cc        | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 db/version_set.cc    | 36 +++++++++++++++++++++++++
 db/version_set.h     |  2 ++
 include/leveldb/db.h |  2 +-
 5 files changed, 115 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index fef085b..42c4952 100644
--- a/Makefile
+++ b/Makefile
@@ -69,7 +69,7 @@ SHARED = $(SHARED1)
 else
 # Update db.h if you change these.
 SHARED_MAJOR = 1
-SHARED_MINOR = 8
+SHARED_MINOR = 9
 SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT)
 SHARED2 = $(SHARED1).$(SHARED_MAJOR)
 SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR)
diff --git a/db/db_test.cc b/db/db_test.cc
index 74abd13..684ea3b 100644
--- a/db/db_test.cc
+++ b/db/db_test.cc
@@ -59,6 +59,12 @@ class SpecialEnv : public EnvWrapper {
   // Simulate non-writable file system while this pointer is non-NULL
   port::AtomicPointer non_writable_;
 
+  // Force sync of manifest files to fail while this pointer is non-NULL
+  port::AtomicPointer manifest_sync_error_;
+
+  // Force write to manifest files to fail while this pointer is non-NULL
+  port::AtomicPointer manifest_write_error_;
+
   bool count_random_reads_;
   AtomicCounter random_read_counter_;
 
@@ -69,6 +75,8 @@ class SpecialEnv : public EnvWrapper {
     no_space_.Release_Store(NULL);
     non_writable_.Release_Store(NULL);
     count_random_reads_ = false;
+    manifest_sync_error_.Release_Store(NULL);
+    manifest_write_error_.Release_Store(NULL);
   }
 
   Status NewWritableFile(const std::string& f, WritableFile** r) {
@@ -100,6 +108,30 @@ class SpecialEnv : public EnvWrapper {
         return base_->Sync();
       }
     };
+    class ManifestFile : public WritableFile {
+     private:
+      SpecialEnv* env_;
+      WritableFile* base_;
+     public:
+      ManifestFile(SpecialEnv* env, WritableFile* b) : env_(env), base_(b) { }
+      ~ManifestFile() { delete base_; }
+      Status Append(const Slice& data) {
+        if (env_->manifest_write_error_.Acquire_Load() != NULL) {
+          return Status::IOError("simulated writer error");
+        } else {
+          return base_->Append(data);
+        }
+      }
+      Status Close() { return base_->Close(); }
+      Status Flush() { return base_->Flush(); }
+      Status Sync() {
+        if (env_->manifest_sync_error_.Acquire_Load() != NULL) {
+          return Status::IOError("simulated sync error");
+        } else {
+          return base_->Sync();
+        }
+      }
+    };
 
     if (non_writable_.Acquire_Load() != NULL) {
       return Status::IOError("simulated write error");
@@ -109,6 +141,8 @@ class SpecialEnv : public EnvWrapper {
     if (s.ok()) {
       if (strstr(f.c_str(), ".sst") != NULL) {
         *r = new SSTableFile(this, *r);
+      } else if (strstr(f.c_str(), "MANIFEST") != NULL) {
+        *r = new ManifestFile(this, *r);
       }
     }
     return s;
@@ -1492,6 +1526,47 @@ TEST(DBTest, NonWritableFileSystem) {
   env_->non_writable_.Release_Store(NULL);
 }
 
+TEST(DBTest, ManifestWriteError) {
+  // Test for the following problem:
+  // (a) Compaction produces file F
+  // (b) Log record containing F is written to MANIFEST file, but Sync() fails
+  // (c) GC deletes F
+  // (d) After reopening DB, reads fail since deleted F is named in log record
+
+  // We iterate twice.  In the second iteration, everything is the
+  // same except the log record never makes it to the MANIFEST file.
+  for (int iter = 0; iter < 2; iter++) {
+    port::AtomicPointer* error_type = (iter == 0)
+        ? &env_->manifest_sync_error_
+        : &env_->manifest_write_error_;
+
+    // Insert foo=>bar mapping
+    Options options = CurrentOptions();
+    options.env = env_;
+    options.create_if_missing = true;
+    options.error_if_exists = false;
+    DestroyAndReopen(&options);
+    ASSERT_OK(Put("foo", "bar"));
+    ASSERT_EQ("bar", Get("foo"));
+
+    // Memtable compaction (will succeed)
+    dbfull()->TEST_CompactMemTable();
+    ASSERT_EQ("bar", Get("foo"));
+    const int last = config::kMaxMemCompactLevel;
+    ASSERT_EQ(NumTableFilesAtLevel(last), 1);   // foo=>bar is now in last level
+
+    // Merging compaction (will fail)
+    error_type->Release_Store(env_);
+    dbfull()->TEST_CompactRange(last, NULL, NULL);  // Should fail
+    ASSERT_EQ("bar", Get("foo"));
+
+    // Recovery: should not lose data
+    error_type->Release_Store(NULL);
+    Reopen(&options);
+    ASSERT_EQ("bar", Get("foo"));
+  }
+}
+
 TEST(DBTest, FilesDeletedAfterCompaction) {
   ASSERT_OK(Put("foo", "v2"));
   Compact("a", "z");
diff --git a/db/version_set.cc b/db/version_set.cc
index bdd4a57..7d0a5de 100644
--- a/db/version_set.cc
+++ b/db/version_set.cc
@@ -786,12 +786,23 @@ Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu) {
       if (s.ok()) {
         s = descriptor_file_->Sync();
       }
+      if (!s.ok()) {
+        Log(options_->info_log, "MANIFEST write: %s\n", s.ToString().c_str());
+        if (ManifestContains(record)) {
+          Log(options_->info_log,
+              "MANIFEST contains log record despite error; advancing to new "
+              "version to prevent mismatch between in-memory and logged state");
+          s = Status::OK();
+        }
+      }
     }
 
     // If we just created a new descriptor file, install it by writing a
     // new CURRENT file that points to it.
     if (s.ok() && !new_manifest_file.empty()) {
       s = SetCurrentFile(env_, dbname_, manifest_file_number_);
+      // No need to double-check MANIFEST in case of error since it
+      // will be discarded below.
     }
 
     mu->Lock();
@@ -1025,6 +1036,31 @@ const char* VersionSet::LevelSummary(LevelSummaryStorage* scratch) const {
   return scratch->buffer;
 }
 
+// Return true iff the manifest contains the specified record.
+bool VersionSet::ManifestContains(const std::string& record) const {
+  std::string fname = DescriptorFileName(dbname_, manifest_file_number_);
+  Log(options_->info_log, "ManifestContains: checking %s\n", fname.c_str());
+  SequentialFile* file = NULL;
+  Status s = env_->NewSequentialFile(fname, &file);
+  if (!s.ok()) {
+    Log(options_->info_log, "ManifestContains: %s\n", s.ToString().c_str());
+    return false;
+  }
+  log::Reader reader(file, NULL, true/*checksum*/, 0);
+  Slice r;
+  std::string scratch;
+  bool result = false;
+  while (reader.ReadRecord(&r, &scratch)) {
+    if (r == Slice(record)) {
+      result = true;
+      break;
+    }
+  }
+  delete file;
+  Log(options_->info_log, "ManifestContains: result = %d\n", result ? 1 : 0);
+  return result;
+}
+
 uint64_t VersionSet::ApproximateOffsetOf(Version* v, const InternalKey& ikey) {
   uint64_t result = 0;
   for (int level = 0; level < config::kNumLevels; level++) {
diff --git a/db/version_set.h b/db/version_set.h
index 792899b..9d084fd 100644
--- a/db/version_set.h
+++ b/db/version_set.h
@@ -277,6 +277,8 @@ class VersionSet {
 
   void AppendVersion(Version* v);
 
+  bool ManifestContains(const std::string& record) const;
+
   Env* const env_;
   const std::string dbname_;
   const Options* const options_;
diff --git a/include/leveldb/db.h b/include/leveldb/db.h
index f1e70a0..29d3674 100644
--- a/include/leveldb/db.h
+++ b/include/leveldb/db.h
@@ -14,7 +14,7 @@ namespace leveldb {
 
 // Update Makefile if you change these
 static const int kMajorVersion = 1;
-static const int kMinorVersion = 8;
+static const int kMinorVersion = 9;
 
 struct Options;
 struct ReadOptions;
-- 
cgit v1.2.3


From bbb0263070defe02ffee97b35d0dc31d3f6297a3 Mon Sep 17 00:00:00 2001
From: "dgrogan@chromium.org"
 
Date: Thu, 7 Feb 2013 02:28:39 +0000
Subject: Make DB::Open fail if sst files are missing.

Also, cleanup for Clang's -Wimplicit-fallthrough warning.


git-svn-id: http://leveldb.googlecode.com/svn/trunk@72 62dab493-f737-651d-591e-8d6aee1b9529
---
 db/db_impl.cc | 16 ++++++++++++----
 db/db_test.cc | 31 +++++++++++++++++++++++++++++++
 util/hash.cc  | 11 +++++++++--
 3 files changed, 52 insertions(+), 6 deletions(-)

diff --git a/db/db_impl.cc b/db/db_impl.cc
index c9de169..058d56d 100644
--- a/db/db_impl.cc
+++ b/db/db_impl.cc
@@ -310,16 +310,24 @@ Status DBImpl::Recover(VersionEdit* edit) {
     if (!s.ok()) {
       return s;
     }
+    std::set expected;
+    versions_->AddLiveFiles(&expected);
     uint64_t number;
     FileType type;
     std::vector logs;
     for (size_t i = 0; i < filenames.size(); i++) {
-      if (ParseFileName(filenames[i], &number, &type)
-          && type == kLogFile
-          && ((number >= min_log) || (number == prev_log))) {
-        logs.push_back(number);
+      if (ParseFileName(filenames[i], &number, &type)) {
+        expected.erase(number);
+        if (type == kLogFile && ((number >= min_log) || (number == prev_log)))
+          logs.push_back(number);
       }
     }
+    if (!expected.empty()) {
+      char buf[50];
+      snprintf(buf, sizeof(buf), "%d missing files; e.g.",
+               static_cast(expected.size()));
+      return Status::Corruption(buf, TableFileName(dbname_, *(expected.begin())));
+    }
 
     // Recover in the order in which the logs were generated
     std::sort(logs.begin(), logs.end());
diff --git a/db/db_test.cc b/db/db_test.cc
index 684ea3b..2f51296 100644
--- a/db/db_test.cc
+++ b/db/db_test.cc
@@ -461,6 +461,20 @@ class DBTest {
     }
     return result;
   }
+
+  bool DeleteAnSSTFile() {
+    std::vector filenames;
+    ASSERT_OK(env_->GetChildren(dbname_, &filenames));
+    uint64_t number;
+    FileType type;
+    for (size_t i = 0; i < filenames.size(); i++) {
+      if (ParseFileName(filenames[i], &number, &type) && type == kTableFile) {
+        ASSERT_OK(env_->DeleteFile(TableFileName(dbname_, number)));
+        return true;
+      }
+    }
+    return false;
+  }
 };
 
 TEST(DBTest, Empty) {
@@ -1567,6 +1581,23 @@ TEST(DBTest, ManifestWriteError) {
   }
 }
 
+TEST(DBTest, MissingSSTFile) {
+  ASSERT_OK(Put("foo", "bar"));
+  ASSERT_EQ("bar", Get("foo"));
+
+  // Dump the memtable to disk.
+  dbfull()->TEST_CompactMemTable();
+  ASSERT_EQ("bar", Get("foo"));
+
+  ASSERT_TRUE(DeleteAnSSTFile());
+  Options options = CurrentOptions();
+  options.paranoid_checks = true;
+  Status s = TryReopen(&options);
+  ASSERT_TRUE(!s.ok());
+  ASSERT_TRUE(s.ToString().find("issing") != std::string::npos)
+      << s.ToString();
+}
+
 TEST(DBTest, FilesDeletedAfterCompaction) {
   ASSERT_OK(Put("foo", "v2"));
   Compact("a", "z");
diff --git a/util/hash.cc b/util/hash.cc
index ba18180..07cf022 100644
--- a/util/hash.cc
+++ b/util/hash.cc
@@ -6,6 +6,13 @@
 #include "util/coding.h"
 #include "util/hash.h"
 
+// The FALLTHROUGH_INTENDED macro can be used to annotate implicit fall-through
+// between switch labels. The real definition should be provided externally.
+// This one is a fallback version for unsupported compilers.
+#ifndef FALLTHROUGH_INTENDED
+#define FALLTHROUGH_INTENDED do { } while (0)
+#endif
+
 namespace leveldb {
 
 uint32_t Hash(const char* data, size_t n, uint32_t seed) {
@@ -28,10 +35,10 @@ uint32_t Hash(const char* data, size_t n, uint32_t seed) {
   switch (limit - data) {
     case 3:
       h += data[2] << 16;
-      // fall through
+      FALLTHROUGH_INTENDED;
     case 2:
       h += data[1] << 8;
-      // fall through
+      FALLTHROUGH_INTENDED;
     case 1:
       h += data[0];
       h *= m;
-- 
cgit v1.2.3