Merge changes I10b3e46d,I29cde728,I383723e5,Idbecb166,I0f70c545

* changes:
  Interceptor: initial analysis tooling
  interceptor: replace JSON log by proto log
  interceptor: add initial proto log format
  interceptor: factor out the command line string creation
  interceptor: introduce default compilation options

7 files changed, 254 insertions, 23 deletions

diff --git a/build-prebuilts.sh b/build-prebuilts.sh
index 4a061c2..8925111 100755
--- a/build-prebuilts.sh
+++ b/build-prebuilts.sh
@@ -37,6 +37,7 @@ EOF
         e2fsdroid
         img2simg
         interceptor
+        interceptor_analysis
         lpmake
         lz4
         mkbootfs
diff --git a/interceptor/Android.bp b/interceptor/Android.bp
index 49d249b..e944816 100644
--- a/interceptor/Android.bp
+++ b/interceptor/Android.bp
@@ -1,16 +1,40 @@
+cc_defaults {
+    name: "interceptor_defaults",
+    static_libs: [
+        "libc++fs",
+        "libinterceptor_log",
+        "libprotobuf-cpp-full",
+    ],
+}
+
 cc_library_host_shared {
     name: "libinterceptor",
     srcs: ["interceptor.cc"],
+    defaults: ["interceptor_defaults"],
     static_libs: [
-        "libc++fs",
         "libbase",
     ],
 }
 
+cc_library_host_static {
+    name: "libinterceptor_log",
+    srcs: ["log.proto"],
+    proto: {
+        type: "full",
+        static: true,
+        export_proto_headers: true,
+        canonical_path_from_root: false,
+    },
+}
+
 cc_binary_host {
     name: "interceptor",
-    srcs: ["main.cc",],
-    static_libs: [
-        "libc++fs",
-    ],
+    srcs: ["main.cc"],
+    defaults: ["interceptor_defaults"],
+}
+
+cc_binary_host {
+    name: "interceptor_analysis",
+    srcs: ["analysis.cc",],
+    defaults: ["interceptor_defaults",],
 }
diff --git a/interceptor/analysis.cc b/interceptor/analysis.cc
new file mode 100644
index 0000000..b7f1535
--- /dev/null
+++ b/interceptor/analysis.cc
@@ -0,0 +1,123 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <getopt.h>
+#include <google/protobuf/text_format.h>
+#include <sysexits.h>
+#include <cstdlib>
+#include <cstring>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <iterator>
+
+#include "log.pb.h"
+
+namespace fs = std::filesystem;
+
+enum class OutputFormat { TEXT };
+
+struct Options {
+  fs::path command_log;
+  OutputFormat output_format = OutputFormat::TEXT;
+  fs::path output;
+};
+
+static Options parse_args(int argc, char* argv[]) {
+  Options result;
+
+  const static option opts[] = {
+      {"command-log", required_argument, nullptr, 'l'},
+      {"output-type", required_argument, nullptr, 't'},
+      {"output", required_argument, nullptr, 'o'},
+      {nullptr, 0, nullptr, 0},
+  };
+  const auto usage = [&]() {
+    std::cerr << "usage: " << argv[0] << '\n'
+              << "  -l|--command-log filename\n"
+              << "  -o|--output filename\n"
+              << " [-t|--output-type (text)]\n";
+    exit(EX_USAGE);
+  };
+  while (true) {
+    int ix;
+    int c = getopt_long(argc, argv, "-l:f:o:", opts, &ix);
+    if (c == -1) break;
+    switch (c) {
+      case 'l':
+        result.command_log = fs::absolute(optarg);
+        break;
+      case 't':
+        if (strcmp(optarg, "text") == 0)
+          result.output_format = OutputFormat::TEXT;
+        else
+          usage();
+        break;
+      case 'o':
+        result.output = fs::absolute(optarg);
+        break;
+      default:
+        usage();
+    }
+  }
+
+  if (result.command_log.empty() || result.output.empty()) {
+    usage();
+  }
+
+  if (!fs::exists(result.command_log)) {
+    std::cerr << "No such file: " << result.command_log << "\n";
+  }
+
+  return result;
+}
+
+interceptor::log::Log read_log(const fs::path& log_file) {
+  interceptor::log::Log result;
+  std::ifstream input(log_file);
+  if (!input) {
+    std::cerr << "Could not open input file for reading.\n";
+    exit(EX_NOINPUT);
+  }
+  result.ParseFromIstream(&input);
+  return result;
+}
+
+void text_to_file(const interceptor::log::Log& log, const fs::path& output) {
+  std::string content;
+  google::protobuf::TextFormat::PrintToString(log, &content);
+  std::ofstream os(output);
+  if (!os) {
+    std::cerr << "Could not open output file for writing.\n";
+    exit(EX_CANTCREAT);
+  }
+  os << content;
+  if (!os.flush()) {
+    std::cerr << "Failed to write to output file.\n";
+    exit(EX_CANTCREAT);
+  }
+}
+
+int main(int argc, char* argv[]) {
+  const auto options = parse_args(argc, argv);
+  const auto log = read_log(options.command_log);
+
+  switch (options.output_format) {
+    case OutputFormat::TEXT:
+      text_to_file(log, options.output);
+      break;
+  }
+}
diff --git a/interceptor/interceptor.cc b/interceptor/interceptor.cc
index caadfa0..f31b343 100644
--- a/interceptor/interceptor.cc
+++ b/interceptor/interceptor.cc
@@ -35,6 +35,7 @@
 #include <utility>
 
 #include <android-base/strings.h>
+#include <google/protobuf/util/delimited_message_util.h>
 
 namespace fs = std::filesystem;
 
@@ -44,7 +45,7 @@ namespace fs = std::filesystem;
 static void process_command(const char* filename, char* const argv[], char* const envp[]);
 
 // log command if logging is enabled
-static void log(const interceptor::Command&, const std::string& prefix);
+static void log(const interceptor::Command&);
 
 // execute potentially modified command
 static void exec(const interceptor::Command&);
@@ -120,17 +121,18 @@ static void dump_vector(std::ostream& os, const char* key, const std::vector<T>&
   os << "]";
 }
 
+std::string Command::command() const {
+  std::ostringstream cmd;
+  cmd << program();
+  if (args().size() > 1) cmd << ' ';
+  std::transform(args().cbegin() + 1, args().cend(), std::ostream_iterator<std::string>(cmd, " "),
+                 escape);
+  return cmd.str();
+}
+
 std::string Command::repr() const {
   std::ostringstream os;
-  os << R"({"cmd": )";
-  {
-    std::ostringstream cmd;
-    cmd << program();
-    if (args().size() > 1) cmd << ' ';
-    std::transform(args().cbegin() + 1, args().cend(), std::ostream_iterator<std::string>(cmd, " "),
-                   escape);
-    os << std::quoted(cmd.str());
-  }
+  os << R"({"cmd": )" << std::quoted(command());
 
   os << ", ";
   dump_vector(os, "in", inputs());
@@ -143,6 +145,19 @@ std::string Command::repr() const {
   return os.str();
 }
 
+log::Message Command::message() const {
+  log::Message result;
+  auto& command = *result.mutable_command();
+
+  command.set_program(program_);
+  *command.mutable_args() = {args().cbegin(), args().cend()};
+  command.set_current_dir(cwd_);
+  *command.mutable_outputs() = {outputs().cbegin(), outputs().cend()};
+  *command.mutable_inputs() = {inputs().cbegin(), inputs().cend()};
+
+  return result;
+}
+
 void Command::make_relative() {
   // determine the ROOT_DIR
   std::string root_dir;
@@ -314,20 +329,21 @@ static void process_command(const char* filename, char* const argv[], char* cons
 
   command.analyze();
 
-  log(command, "");
+  log(command);
 
   // pass down the transformed command to execve
   exec(command);
 }
 
-static void log(const interceptor::Command& command, const std::string& prefix) {
+static void log(const interceptor::Command& command) {
   const auto& env = command.env();
 
   if (const auto env_it = env.find(ENV_command_log); env_it != env.cend()) {
     std::ofstream file;
-    file.open(std::string(env_it->second), std::ofstream::out | std::ofstream::app);
+    file.open(std::string(env_it->second),
+              std::ofstream::out | std::ofstream::app | std::ofstream::binary);
     if (file.is_open()) {
-      file << prefix << command.repr() << ",\n";
+      google::protobuf::util::SerializeDelimitedToOstream(command.message(), &file);
     }
   }
 }
diff --git a/interceptor/interceptor.h b/interceptor/interceptor.h
index d7a38a9..9e9a1e8 100644
--- a/interceptor/interceptor.h
+++ b/interceptor/interceptor.h
@@ -20,6 +20,8 @@
 #include <unordered_map>
 #include <vector>
 
+#include "log.pb.h"
+
 // Options passed via environment variables from the interceptor starter
 constexpr static auto ENV_command_log = "INTERCEPTOR_command_log";
 constexpr static auto ENV_root_dir = "INTERCEPTOR_root_dir";
@@ -51,6 +53,7 @@ class Command {
   const Outputs& outputs() const { return outputs_; }
 
   std::string repr() const;
+  log::Message message() const;
 
   // make command line calls relative to ROOT_DIR
   void make_relative();
@@ -59,6 +62,8 @@ class Command {
   void analyze();
 
  private:
+  std::string command() const;
+
   std::string program_;
   std::string cwd_;
 
diff --git a/interceptor/log.proto b/interceptor/log.proto
new file mode 100644
index 0000000..fc9e31d
--- /dev/null
+++ b/interceptor/log.proto
@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+syntax = "proto3";
+
+package interceptor.log;
+
+// A Command as traced by intercepting an execve() invocation.
+message Command {
+  string program = 1;
+  repeated string args = 2;
+  string current_dir = 3;
+  repeated string inputs = 4;
+  repeated string outputs = 5;
+};
+
+// A single message as emitted by an intercepted process. It can contain any of
+// the above messages.
+message Message {
+  oneof message { Command command = 1; }
+}
+
+// The entirety of a the final log should be stored as just this Log.
+message Log {
+  repeated Command commands = 1;
+}
diff --git a/interceptor/main.cc b/interceptor/main.cc
index ae3d623..e9184e6 100644
--- a/interceptor/main.cc
+++ b/interceptor/main.cc
@@ -16,6 +16,7 @@
 
 #include <getopt.h>
 #include <stdlib.h>
+#include <sysexits.h>
 #include <cstdlib>
 #include <filesystem>
 #include <fstream>
@@ -23,6 +24,8 @@
 #include <optional>
 #include <sstream>
 
+#include <google/protobuf/util/delimited_message_util.h>
+
 #include "interceptor.h"
 
 namespace fs = std::filesystem;
@@ -79,7 +82,7 @@ static void setup_interceptor_library_path() {
     interceptor_library = fs::read_symlink(interceptor_library);
   if (!fs::is_regular_file(interceptor_library)) {
     std::cerr << "Interceptor library could not be found!\n";
-    exit(1);
+    exit(EX_CONFIG);
   }
   setenv("LD_PRELOAD", interceptor_library.c_str(), 1);
 }
@@ -101,14 +104,32 @@ class CommandLog {
     if (command_log_file_) {
       setenv(ENV_command_log, command_log_file_->c_str(), 1);
       std::ofstream command_log(command_log_file_->c_str(), std::ios_base::trunc);
-      command_log << "[\n";
+      if (!command_log) {
+        std::cerr << "Could not open command log for writing: " << *command_log_file_ << "\n";
+        exit(EX_CANTCREAT);
+      }
     }
   }
 
   ~CommandLog() {
     if (command_log_file_) {
-      std::ofstream command_log(command_log_file_->c_str(), std::ios_base::app);
-      command_log << "]\n";
+      // compact the log by re-reading the individual log::Message's to combine
+      // them to a log::Log
+      interceptor::log::Log log;
+      {
+        std::ifstream command_log(command_log_file_->c_str(), std::ios_base::binary);
+
+        google::protobuf::io::IstreamInputStream input_stream(&command_log);
+        interceptor::log::Message message;
+        while (true) {
+          if (!google::protobuf::util::ParseDelimitedFromZeroCopyStream(&message, &input_stream,
+                                                                        nullptr))
+            break;
+          if (message.has_command()) log.add_commands()->Swap(message.release_command());
+        }
+      }
+      std::ofstream command_log(command_log_file_->c_str(), std::ios_base::binary);
+      log.SerializeToOstream(&command_log);
     }
   }
 };
@@ -121,5 +142,7 @@ int main(int argc, char* argv[]) {
 
   CommandLog command_log(options.command_log);
 
+  // TODO: cleanly to google::protobuf::ShutdownProtobufLibrary();
+
   return std::system(options.command_line.c_str());
 }