diff options
| author | Xin Li <delphij@google.com> | 2023-08-14 15:38:11 -0700 |
|---|---|---|
| committer | Xin Li <delphij@google.com> | 2023-08-14 15:38:11 -0700 |
| commit | 97bad90ab4956a5c8172692291ba16cb538f0fcb (patch) | |
| tree | 58860f9e332410ab89db0730585820af6ce5c448 | |
| parent | 953e117c850b8ccae72bd2996e0b1ce94f019e05 (diff) | |
| parent | 9058c22d3f5f5bd9162c7ecc24402187375adae9 (diff) | |
| download | mobile-data-download-97bad90ab4956a5c8172692291ba16cb538f0fcb.tar.gz | |
Merge Android U (ab/10368041)
Bug: 291102124
Merged-In: I334c003ca53cdf7d9be88f36721a7116afd7b37a
Change-Id: I67657e16808ead0d6c208d3e2284503558067c82
262 files changed, 16645 insertions, 4696 deletions
@@ -34,6 +34,47 @@ java_library { srcs: ["android-annotation-stubs/src/**/*.java"], host_supported: true, sdk_version: "core_current", + apex_available: [ + "//apex_available:platform", + "com.android.adservices", + "com.android.extservices", + "com.android.ondevicepersonalization", + ], +} + +android_library { + name: "mdd-robolectric-library", + srcs: [ + "javatests/com/google/android/libraries/mobiledatadownload/internal/MddTestUtil.java", + "javatests/com/google/android/libraries/mobiledatadownload/testing/**/*.java", + "java/com/google/android/libraries/mobiledatadownload/file/common/testing/FakeFileBackend.java", + "java/com/google/android/libraries/mobiledatadownload/file/common/testing/TemporaryUri.java", + ], + exclude_srcs: [ + // TODO: (b/256877824) to be removed once RunfilesPaths is imported. + // The current test cases are not referencing on these classes. + "javatests/com/google/android/libraries/mobiledatadownload/testing/RobolectricFileDownloader.java", // Missing RunfilesPaths + "javatests/com/google/android/libraries/mobiledatadownload/testing/MddNotificationCapture.java", // Missing GoogleLogger, AndroidTestUtil + "javatests/com/google/android/libraries/mobiledatadownload/testing/BlockingFileDownloader.java", // Missing GoogleLogger + "javatests/com/google/android/libraries/mobiledatadownload/testing/FakeMobileDataDownload.java", // Missing GoogleLogger + "javatests/com/google/android/libraries/mobiledatadownload/testing/MddTestDependencies.java", // Missing BaseFileDownloaderModule + "javatests/com/google/android/libraries/mobiledatadownload/internal/ExpirationHandlerTest.java" // Test failed + ], + + libs: [ + "ub-uiautomator", + "androidx.test.ext.truth", + "androidx.test.rules", + "androidx.annotation_annotation", + "org.apache.http.legacy", + "mobile_data_downloader_lib", + "auto_value_annotations", + "framework-annotations-lib", + "checker-qual", + ], + visibility: [ + ":__subpackages__", + ], } android_library { @@ -42,9 +83,8 @@ android_library { "java/**/*.java", ], exclude_srcs: [ - "java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/**/*.java", - "java/com/google/android/libraries/mobiledatadownload/file/common/testing/**/*.java", - "java/com/google/android/libraries/mobiledatadownload/downloader/offroad/Offroad2FileDownloader.java", + "java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/**/*.java", + "java/com/google/android/libraries/mobiledatadownload/file/common/testing/**/*.java", ], static_libs: [ "androidx.core_core", @@ -55,14 +95,14 @@ android_library { "mobile-data-download-populator-java-proto-lite", "dagger2", "jsr330", - "checker-qual", "android_downloader_lib", + "android_checker_annotation_stubs", ], libs: [ "auto_value_annotations", "framework-annotations-lib", "unsupportedappusage", - "android_checker_annotation_stubs", + "checker-qual", ], plugins: [ "auto_value_plugin", @@ -74,13 +114,17 @@ android_library { apex_available: [ "//apex_available:platform", "com.android.adservices", + "com.android.extservices", + "com.android.ondevicepersonalization", ], visibility: [ "//packages/modules/AdServices:__subpackages__", + "//packages/modules/OnDevicePersonalization:__subpackages__", + ":__subpackages__", ], errorprone: { javacflags: [ "-Xep:NoCanIgnoreReturnValueOnClasses:WARN", ], }, -}
\ No newline at end of file +} diff --git a/java/com/google/android/libraries/mobiledatadownload/AggregateException.java b/java/com/google/android/libraries/mobiledatadownload/AggregateException.java index 94080d9..117918a 100644 --- a/java/com/google/android/libraries/mobiledatadownload/AggregateException.java +++ b/java/com/google/android/libraries/mobiledatadownload/AggregateException.java @@ -175,7 +175,7 @@ public final class AggregateException extends Exception { @VisibleForTesting static String throwableToString(Throwable failure) { - return throwableToString(failure, /*depth=*/ 1); + return throwableToString(failure, /* depth= */ 1); } private static String throwableToString(Throwable failure, int depth) { diff --git a/java/com/google/android/libraries/mobiledatadownload/BUILD b/java/com/google/android/libraries/mobiledatadownload/BUILD index 733d814..ca39a4e 100644 --- a/java/com/google/android/libraries/mobiledatadownload/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/BUILD @@ -13,7 +13,10 @@ # limitations under the License. load("@build_bazel_rules_android//android:rules.bzl", "android_library") +# MDI download (MDD) visibility is restricted to the following set of packages. Any +# new clients must be added to this list in order to grant build visibility. package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -22,27 +25,22 @@ package( android_library( name = "mobiledatadownload", - srcs = glob( - ["*.java"], - exclude = [ - "AccountSource.java", - "AggregateException.java", - "Configurator.java", - "TimeSource.java", - "Flags.java", - "Constants.java", - "DownloadException.java", - "DownloadListener.java", - "Logger.java", - "MobileDataDownloadBuilder.java", - "SilentFeedback.java", - "UsageEvent.java", - "SingleFileDownloadRequest.java", - "SingleFileDownloadListener.java", - "FileSource.java", - "ExperimentationConfig.java", - ], - ), + srcs = [ + "AddFileGroupRequest.java", + "CustomFileGroupValidator.java", + "DownloadFileGroupRequest.java", + "FileGroupPopulator.java", + "GetFileGroupRequest.java", + "GetFileGroupsByFilterRequest.java", + "ImportFilesRequest.java", + "MobileDataDownload.java", + "MobileDataDownloadImpl.java", + "ReadDataFileGroupRequest.java", + "RemoveFileGroupRequest.java", + "RemoveFileGroupsByFilterRequest.java", + "RemoveFileGroupsByFilterResponse.java", + "TaskScheduler.java", + ], exports = [ ":single_file_interfaces", ], @@ -51,22 +49,32 @@ android_library( ":DownloadListener", ":FileSource", ":Flags", + ":TimeSource", ":UsageEvent", ":single_file_interfaces", "//java/com/google/android/libraries/mobiledatadownload/account:AccountUtil", "//java/com/google/android/libraries/mobiledatadownload/file", + "//java/com/google/android/libraries/mobiledatadownload/foreground:ForegroundDownloadKey", "//java/com/google/android/libraries/mobiledatadownload/foreground:NotificationUtil", + "//java/com/google/android/libraries/mobiledatadownload/internal:DownloadGroupState", + "//java/com/google/android/libraries/mobiledatadownload/internal:ExceptionToMddResultMapper", + "//java/com/google/android/libraries/mobiledatadownload/internal:MddConstants", "//java/com/google/android/libraries/mobiledatadownload/internal:MobileDataDownloadManager", + "//java/com/google/android/libraries/mobiledatadownload/internal/collect", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "//java/com/google/android/libraries/mobiledatadownload/internal/util:DownloadFutureMap", "//java/com/google/android/libraries/mobiledatadownload/internal/util:MddLiteConversionUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/util:ProtoConversionUtil", "//java/com/google/android/libraries/mobiledatadownload/lite", "//java/com/google/android/libraries/mobiledatadownload/monitor:DownloadProgressMonitor", "//java/com/google/android/libraries/mobiledatadownload/tracing", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "//proto:client_config_java_proto_lite", "//proto:download_config_java_proto_lite", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "@androidx_annotation_annotation", "@androidx_core_core", "@com_google_auto_value", @@ -86,20 +94,16 @@ android_library( ":AccountSource", ":Configurator", ":Constants", - ":DownloadException", - ":DownloadListener", ":ExperimentationConfig", ":Flags", ":Logger", ":SilentFeedback", ":mobiledatadownload", "//java/com/google/android/libraries/mobiledatadownload/account:AccountManagerAccountSource", - "//java/com/google/android/libraries/mobiledatadownload/account:AccountUtil", "//java/com/google/android/libraries/mobiledatadownload/delta:DeltaDecoder", "//java/com/google/android/libraries/mobiledatadownload/downloader:FileDownloader", "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/foreground:NotificationUtil", - "//java/com/google/android/libraries/mobiledatadownload/internal:MobileDataDownloadManager", "//java/com/google/android/libraries/mobiledatadownload/internal/dagger:ApplicationContextModule", "//java/com/google/android/libraries/mobiledatadownload/internal/dagger:DownloaderModule", "//java/com/google/android/libraries/mobiledatadownload/internal/dagger:ExecutorsModule", @@ -111,19 +115,17 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/internal/logging:MddEventLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:NoOpEventLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", - "//java/com/google/android/libraries/mobiledatadownload/internal/util:ProtoConversionUtil", "//java/com/google/android/libraries/mobiledatadownload/lite", "//java/com/google/android/libraries/mobiledatadownload/monitor:DownloadProgressMonitor", "//java/com/google/android/libraries/mobiledatadownload/monitor:NetworkUsageMonitor", - "//java/com/google/android/libraries/mobiledatadownload/tracing", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "//proto:client_config_java_proto_lite", "//proto:download_config_java_proto_lite", + "//proto:logs_java_proto_lite", "@androidx_core_core", "@com_google_auto_value", - "@com_google_code_findbugs_jsr305", "@com_google_dagger", "@com_google_guava_guava", - "@com_google_protobuf//:protobuf_lite", ], ) @@ -199,7 +201,10 @@ android_library( android_library( name = "DownloadException", srcs = ["DownloadException.java"], - deps = ["@com_google_guava_guava"], + deps = [ + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "@com_google_guava_guava", + ], ) android_library( @@ -241,6 +246,7 @@ android_library( ], deps = [ "//proto:client_config_java_proto_lite", + "//proto:log_enums_java_proto_lite", "@com_google_auto_value", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/Constants.java b/java/com/google/android/libraries/mobiledatadownload/Constants.java index 7c71cd1..7b234b9 100644 --- a/java/com/google/android/libraries/mobiledatadownload/Constants.java +++ b/java/com/google/android/libraries/mobiledatadownload/Constants.java @@ -36,7 +36,7 @@ public final class Constants { /** The version of MDD library. Same as mdi_download module version. */ // TODO(b/122271766): Figure out how to update this automatically. // LINT.IfChange - public static final int MDD_LIB_VERSION = 422883838; + public static final int MDD_LIB_VERSION = 516938429; // LINT.ThenChange(<internal>) // <internal> diff --git a/java/com/google/android/libraries/mobiledatadownload/DownloadException.java b/java/com/google/android/libraries/mobiledatadownload/DownloadException.java index cc9a148..43f8659 100644 --- a/java/com/google/android/libraries/mobiledatadownload/DownloadException.java +++ b/java/com/google/android/libraries/mobiledatadownload/DownloadException.java @@ -17,10 +17,11 @@ package com.google.android.libraries.mobiledatadownload; import static com.google.common.util.concurrent.Futures.immediateFailedFuture; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import com.google.errorprone.annotations.CanIgnoreReturnValue; /** Thrown when there is a download failure. */ public final class DownloadException extends Exception { @@ -171,18 +172,21 @@ public final class DownloadException extends Exception { private Throwable cause; /** Sets the {@link DownloadResultCode}. */ + @CanIgnoreReturnValue public Builder setDownloadResultCode(DownloadResultCode downloadResultCode) { this.downloadResultCode = downloadResultCode; return this; } /** Sets the error message. */ + @CanIgnoreReturnValue public Builder setMessage(String message) { this.message = message; return this; } /** Sets the cause of the exception. */ + @CanIgnoreReturnValue public Builder setCause(Throwable cause) { this.cause = cause; return this; @@ -213,7 +217,7 @@ public final class DownloadException extends Exception { */ public static <T> ListenableFuture<T> wrapIfFailed( ListenableFuture<T> future, DownloadResultCode code, String message) { - return Futures.catchingAsync( + return PropagatedFutures.catchingAsync( future, Throwable.class, (Throwable t) -> immediateFailedFuture(wrap(t, code, message)), diff --git a/java/com/google/android/libraries/mobiledatadownload/DownloadFileGroupRequest.java b/java/com/google/android/libraries/mobiledatadownload/DownloadFileGroupRequest.java index 8b98527..63c337c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/DownloadFileGroupRequest.java +++ b/java/com/google/android/libraries/mobiledatadownload/DownloadFileGroupRequest.java @@ -27,12 +27,10 @@ import javax.annotation.concurrent.Immutable; public abstract class DownloadFileGroupRequest { /** Defines notifiction behavior for foreground download requests. */ - // LINT.IfChange(show_notifications) public enum ShowNotifications { NONE, ALL, } - // LINT.ThenChange(<internal>) DownloadFileGroupRequest() {} @@ -81,11 +79,16 @@ public abstract class DownloadFileGroupRequest { public abstract boolean preserveZipDirectories(); + public abstract boolean verifyIsolatedStructure(); + + public abstract Builder toBuilder(); + public static Builder newBuilder() { return new AutoValue_DownloadFileGroupRequest.Builder() .setGroupSizeBytes(0) .setShowNotifications(ShowNotifications.ALL) - .setPreserveZipDirectories(false); + .setPreserveZipDirectories(false) + .setVerifyIsolatedStructure(true); } /** Builder for {@link DownloadFileGroupRequest}. */ @@ -154,6 +157,21 @@ public abstract class DownloadFileGroupRequest { */ public abstract Builder setPreserveZipDirectories(boolean preserve); + /** + * By default, file groups will isolated structures will have this structure checked for each + * file when returning the file group. If the isolated structure is not correct, MDD will return + * a failure. + * + * <p>Setting this option to false allows clients to bypass this check, reducing the latency for + * critical callpaths. + * + * <p>For groups that do not have an isolated structure, this option is a no-op. + * + * <p>NOTE: All groups with isolated structures are also verified/fixed during MDD's maintenance + * periodic task. + */ + public abstract Builder setVerifyIsolatedStructure(boolean verifyIsolatedStructure); + public abstract DownloadFileGroupRequest build(); } } diff --git a/java/com/google/android/libraries/mobiledatadownload/DownloadListener.java b/java/com/google/android/libraries/mobiledatadownload/DownloadListener.java index 240406d..673bfc7 100644 --- a/java/com/google/android/libraries/mobiledatadownload/DownloadListener.java +++ b/java/com/google/android/libraries/mobiledatadownload/DownloadListener.java @@ -42,8 +42,14 @@ public interface DownloadListener { * * <p>The onComplete is run on MDD Control Executor. If you need to do heavy work, please offload * to a background task. + * + * <p>If using foreground downloads, an exception may be thrown here to tell MDD a failure + * notification should be shown instead of a success notification. <b>NOTE:</b> this is the only + * case where the exception will be taken into account. Throwing an exception here will + * <em>NOT</em> cause the download future returned by MDD to fail. */ - void onComplete(ClientFileGroup clientFileGroup); + // TODO (b/236401280): Switch to async api + void onComplete(ClientFileGroup clientFileGroup) throws Exception; /** This will be called when the download failed. */ default void onFailure(Throwable t) { diff --git a/java/com/google/android/libraries/mobiledatadownload/Flags.java b/java/com/google/android/libraries/mobiledatadownload/Flags.java index 6a5bead..1af1cb2 100644 --- a/java/com/google/android/libraries/mobiledatadownload/Flags.java +++ b/java/com/google/android/libraries/mobiledatadownload/Flags.java @@ -141,6 +141,7 @@ public interface Flags { return true; } + /** Controls whether daily maintenance includes {@link MobileDataDownload#collectGarbage}. */ default boolean mddEnableGarbageCollection() { return true; } @@ -184,10 +185,20 @@ public interface Flags { } default boolean enableRngBasedDeviceStableSampling() { - return false; // TODO(b/144684763): Switch to true after fully rolled out. + return true; + } + + /** + * Controls the key used for file download deduping. + * + * <p>By default, this flag is FALSE, so file download deduping is performed using the destination + * file uri. If this flag is enabled (TRUE), file download deduping will use NewFileKey. + */ + default boolean enableFileDownloadDedupByFileKey() { + return false; } - // PeriodTaskFlags + // PeriodicTaskFlags default long maintenanceGcmTaskPeriod() { return 86400; } diff --git a/java/com/google/android/libraries/mobiledatadownload/GetFileGroupRequest.java b/java/com/google/android/libraries/mobiledatadownload/GetFileGroupRequest.java index bf117d5..05cabf8 100644 --- a/java/com/google/android/libraries/mobiledatadownload/GetFileGroupRequest.java +++ b/java/com/google/android/libraries/mobiledatadownload/GetFileGroupRequest.java @@ -34,8 +34,12 @@ public abstract class GetFileGroupRequest { public abstract boolean preserveZipDirectories(); + public abstract boolean verifyIsolatedStructure(); + public static Builder newBuilder() { - return new AutoValue_GetFileGroupRequest.Builder().setPreserveZipDirectories(false); + return new AutoValue_GetFileGroupRequest.Builder() + .setPreserveZipDirectories(false) + .setVerifyIsolatedStructure(true); } /** Builder for {@link GetFileGroupRequest}. */ @@ -60,6 +64,21 @@ public abstract class GetFileGroupRequest { */ public abstract Builder setPreserveZipDirectories(boolean preserve); + /** + * By default, file groups will isolated structures will have this structure checked for each + * file when returning the file group. If the isolated structure is not correct, MDD will return + * a failure. + * + * <p>Setting this option to false allows clients to bypass this check, reducing the latency for + * critical callpaths. + * + * <p>For groups that do not have an isolated structure, this option is a no-op. + * + * <p>NOTE: All groups with isolated structures are also verified/fixed during MDD's maintenance + * periodic task. + */ + public abstract Builder setVerifyIsolatedStructure(boolean verifyIsolatedStructure); + public abstract GetFileGroupRequest build(); } } diff --git a/java/com/google/android/libraries/mobiledatadownload/GetFileGroupsByFilterRequest.java b/java/com/google/android/libraries/mobiledatadownload/GetFileGroupsByFilterRequest.java index 504ddf7..2901074 100644 --- a/java/com/google/android/libraries/mobiledatadownload/GetFileGroupsByFilterRequest.java +++ b/java/com/google/android/libraries/mobiledatadownload/GetFileGroupsByFilterRequest.java @@ -41,11 +41,14 @@ public abstract class GetFileGroupsByFilterRequest { public abstract boolean preserveZipDirectories(); + public abstract boolean verifyIsolatedStructure(); + public static Builder newBuilder() { return new AutoValue_GetFileGroupsByFilterRequest.Builder() .setIncludeAllGroups(false) .setGroupWithNoAccountOnly(false) - .setPreserveZipDirectories(false); + .setPreserveZipDirectories(false) + .setVerifyIsolatedStructure(true); } /** Builder for {@link GetFileGroupsByFilterRequest}. */ @@ -76,6 +79,21 @@ public abstract class GetFileGroupsByFilterRequest { */ public abstract Builder setPreserveZipDirectories(boolean preserve); + /** + * By default, file groups will isolated structures will have this structure checked for each + * file when returning the file group. If the isolated structure is not correct, MDD will return + * a failure. + * + * <p>Setting this option to false allows clients to bypass this check, reducing the latency for + * critical callpaths. + * + * <p>For groups that do not have an isolated structure, this option is a no-op. + * + * <p>NOTE: All groups with isolated structures are also verified/fixed during MDD's maintenance + * periodic task. + */ + public abstract Builder setVerifyIsolatedStructure(boolean verifyIsolatedStructure); + abstract GetFileGroupsByFilterRequest autoBuild(); public final GetFileGroupsByFilterRequest build() { @@ -84,6 +102,7 @@ public abstract class GetFileGroupsByFilterRequest { if (getFileGroupsByFilterRequest.includeAllGroups()) { checkArgument(!getFileGroupsByFilterRequest.groupNameOptional().isPresent()); checkArgument(!getFileGroupsByFilterRequest.accountOptional().isPresent()); + checkArgument(!getFileGroupsByFilterRequest.groupWithNoAccountOnly()); } else { checkArgument( getFileGroupsByFilterRequest.groupNameOptional().isPresent(), diff --git a/java/com/google/android/libraries/mobiledatadownload/MobileDataDownload.java b/java/com/google/android/libraries/mobiledatadownload/MobileDataDownload.java index 688691e..1894e86 100644 --- a/java/com/google/android/libraries/mobiledatadownload/MobileDataDownload.java +++ b/java/com/google/android/libraries/mobiledatadownload/MobileDataDownload.java @@ -18,10 +18,12 @@ package com.google.android.libraries.mobiledatadownload; import com.google.android.libraries.mobiledatadownload.TaskScheduler.ConstraintOverrides; import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.errorprone.annotations.CheckReturnValue; import com.google.mobiledatadownload.ClientConfigProto.ClientFileGroup; +import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; import java.util.Map; /** The root object and entry point for the MobileDataDownload library. */ @@ -80,6 +82,18 @@ public interface MobileDataDownload { RemoveFileGroupsByFilterRequest removeFileGroupsByFilterRequest); /** + * Gets the file group definition that was added to MDD. This API cannot be used to access files, + * but it can be accessed by populators to manipulate the existing file group state - eg, to + * rename a file group, or otherwise migrate from one format to another. + * + * @return DataFileGroup if downloaded file group is found, otherwise a failing LF. + */ + default ListenableFuture<DataFileGroup> readDataFileGroup( + ReadDataFileGroupRequest readDataFileGroupRequest) { + throw new UnsupportedOperationException(); + } + + /** * Returns the latest downloaded data that we have for the given group name. * * <p>This api takes an instance of {@link GetFileGroupRequest} that contains group name, and it @@ -88,6 +102,10 @@ public interface MobileDataDownload { * <p>This listenable future will return null if no group exists or has been downloaded for the * given group name. * + * <p>Note: getFileGroup returns a snapshot of the latest state, but it's possible for the state + * to change between a getFileGroup call and accessing the files if the ClientFileGroup gets + * cached. Caching the returned ClientFileGroup is therefore discouraged. + * * @param getFileGroupRequest The request to get a single file group. * @return The ListenableFuture of requested client file group for the given request. */ @@ -102,6 +120,10 @@ public interface MobileDataDownload { * filtering, i.e. when no account is specified in the filter, file groups won't be filtered based * on account. * + * <p>Note: getFileGroupsByFilter returns a snapshot of the latest state, but it's possible for + * the state to change between a getFileGroupsByFilter call and accessing the files if the + * ClientFileGroup gets cached. Caching the returned ClientFileGroup is therefore discouraged. + * * @param getFileGroupsByFilterRequest The request to get multiple file groups after filtering. * @return The ListenableFuture that will resolve to a list of the requested client file groups, * including pending and downloaded versions; this ListenableFuture will resolve to all client @@ -227,8 +249,6 @@ public interface MobileDataDownload { * * @param downloadFileGroupRequest The request to download file group. */ - // TODO: Handle the case where a client calls this API for the same group when the - // earlier call has not finished. ListenableFuture<ClientFileGroup> downloadFileGroup( DownloadFileGroupRequest downloadFileGroupRequest); @@ -302,13 +322,15 @@ public interface MobileDataDownload { * <p>Attempts to cancel an on-going foreground download using best effort. If download is unknown * to MDD, this operation is a noop. * - * <p>If the download was started with {@link - * #downloadFileGroupWithForegroundService(DownloadFileGroupRequest)}, the specific {@code - * downloadKey} must be the group name of the file group. + * <p>The key passed here must be created using {@link ForegroundDownloadKey}, and must match the + * properties used from the request. Depending on which API was used to start the download, this + * would be {@link DownloadFileGroupRequest} for {@link SingleFileDownloadRequest}. * - * <p>If the download was started with {@link - * #downloadFileWithForegroundService(SingleFileDownloadRequest)}, the specific {@code - * downloadKey} must be the destination file uri (in string form). + * <p><b>NOTE:</b> In most cases, clients will not need to call this -- it is meant to allow the + * ForegroundDownloadService to cancel a download via the Cancel action registered to a + * notification. + * + * <p>Clients should prefer to cancel the future returned to them from the download call. * * @param downloadKey the key associated with the download */ @@ -328,6 +350,16 @@ public interface MobileDataDownload { ListenableFuture<Void> maintenance(); /** + * Perform garbage collection, which includes removing expired file groups and unreferenced files. + * + * <p>By default, this is run as part of {@link #maintenance} so doesn't need to be invoked + * directly by client code. If you disabled that behavior via {@link + * Flags#mddEnableGarbageCollection} then this method should be periodically called to clean up + * unused files. + */ + ListenableFuture<Void> collectGarbage(); + + /** * Schedule periodic tasks that will download and verify all file groups when the required * conditions are met, using the given {@link TaskScheduler}. * @@ -376,6 +408,18 @@ public interface MobileDataDownload { Optional<Map<String, ConstraintOverrides>> constraintOverridesMap); /** + * Cancels previously-scheduled periodic background tasks using the given {@link TaskScheduler}. + * Cancelling is best-effort and only meant to be used in an emergency; most apps will never need + * to call it. + * + * <p>If the host app doesn't provide a TaskScheduler, calling this API is a no-op. + */ + default ListenableFuture<Void> cancelPeriodicBackgroundTasks() { + // TODO(b/223822302): remove default once all implementations have been updated to include it + return Futures.immediateVoidFuture(); + } + + /** * Handle a task scheduled via a task scheduling service. * * <p>This method should not be called on the main thread, as it does work on the thread it is diff --git a/java/com/google/android/libraries/mobiledatadownload/MobileDataDownloadBuilder.java b/java/com/google/android/libraries/mobiledatadownload/MobileDataDownloadBuilder.java index 5cfb0eb..931dbac 100644 --- a/java/com/google/android/libraries/mobiledatadownload/MobileDataDownloadBuilder.java +++ b/java/com/google/android/libraries/mobiledatadownload/MobileDataDownloadBuilder.java @@ -34,22 +34,25 @@ import com.google.android.libraries.mobiledatadownload.internal.logging.NoOpEven import com.google.android.libraries.mobiledatadownload.lite.Downloader; import com.google.android.libraries.mobiledatadownload.monitor.DownloadProgressMonitor; import com.google.android.libraries.mobiledatadownload.monitor.NetworkUsageMonitor; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.security.SecureRandom; import java.util.ArrayList; import java.util.List; import java.util.concurrent.Executor; /** - * A Builder for the {@link MobileDataDownload}. + * A builder for {@link MobileDataDownload}. + * + * <p> * * <p>WARNING: Only one object should be built. Otherwise, there may be locking errors on the * underlying database and unnecessary memory consumption. @@ -89,6 +92,7 @@ public final class MobileDataDownloadBuilder { componentBuilder = DaggerStandaloneComponent.builder(); } + @CanIgnoreReturnValue public MobileDataDownloadBuilder setContext(Context context) { this.context = context.getApplicationContext(); return this; @@ -103,6 +107,7 @@ public final class MobileDataDownloadBuilder { * directory, and periodic backbround tasks. There is no sharing and no-dedup between instances. * Please talk to <internal>@ before using this. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setInstanceIdOptional(Optional<String> instanceIdOptional) { this.instanceIdOptional = instanceIdOptional; return this; @@ -114,6 +119,7 @@ public final class MobileDataDownloadBuilder { * <p>NOTE: Control Executor must not be single thread executor otherwise it could lead to * deadlock or other side effects. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setControlExecutor(ListeningExecutorService controlExecutor) { Preconditions.checkNotNull(controlExecutor); // Executor that will execute tasks sequentially. @@ -127,6 +133,7 @@ public final class MobileDataDownloadBuilder { * <p>If this is not set, then the client is responsible for refreshing the list of file groups in * MDD as and when they see fit. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder addFileGroupPopulator(FileGroupPopulator fileGroupPopulator) { this.fileGroupPopulatorList.add(fileGroupPopulator); return this; @@ -139,6 +146,7 @@ public final class MobileDataDownloadBuilder { * <p>If this is not set, then the client is responsible for refreshing the list of file groups in * MDD as and when they see fit. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder addFileGroupPopulators( ImmutableList<FileGroupPopulator> fileGroupPopulators) { this.fileGroupPopulatorList.addAll(fileGroupPopulators); @@ -150,24 +158,28 @@ public final class MobileDataDownloadBuilder { * can use GCM, FJD or Work Manager to schedule tasks, and then forward the notification to {@link * MobileDataDownload#handleTask(String)}. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setTaskScheduler(Optional<TaskScheduler> taskSchedulerOptional) { this.taskSchedulerOptional = taskSchedulerOptional; return this; } /** Set the optional Configurator which if present will be used by MDD to configure its flags. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setConfiguratorOptional(Optional<Configurator> configurator) { this.configurator = configurator; return this; } /** Set the optional Logger which if present will be used by MDD to log events. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setLoggerOptional(Optional<Logger> logger) { this.loggerOptional = logger; return this; } /** Set the flags otherwise default values will be used only. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setFlagsOptional(Optional<Flags> flags) { this.flagsOptional = flags; return this; @@ -176,6 +188,7 @@ public final class MobileDataDownloadBuilder { /** * Set the optional SilentFeedback which if present will be used by MDD to send silent feedbacks. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setSilentFeedbackOptional( Optional<SilentFeedback> silentFeedbackOptional) { this.silentFeedbackOptional = silentFeedbackOptional; @@ -186,6 +199,7 @@ public final class MobileDataDownloadBuilder { * Set the MobStore SynchronousFileStorage. Ideally this should be the same object as the one used * by the client app to read files from MDD */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setFileStorage(SynchronousFileStorage fileStorage) { this.fileStorage = fileStorage; return this; @@ -195,6 +209,7 @@ public final class MobileDataDownloadBuilder { * Set the NetworkUsageMonitor. This NetworkUsageMonitor instance must be the same instance that * is registered with SynchronousFileStorage. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setNetworkUsageMonitor(NetworkUsageMonitor networkUsageMonitor) { this.networkUsageMonitor = networkUsageMonitor; return this; @@ -204,6 +219,7 @@ public final class MobileDataDownloadBuilder { * Set the DownloadProgressMonitor. This DownloadProgressMonitor instance must be the same * instance that is registered with SynchronousFileStorage. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setDownloadMonitorOptional( Optional<DownloadProgressMonitor> downloadMonitorOptional) { this.downloadMonitorOptional = downloadMonitorOptional; @@ -214,6 +230,7 @@ public final class MobileDataDownloadBuilder { * Set the FileDownloader Supplier. MDD takes in a Supplier of FileDownload to support lazy * instantiation of the FileDownloader */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setFileDownloaderSupplier( Supplier<FileDownloader> fileDownloaderSupplier) { this.fileDownloaderSupplier = fileDownloaderSupplier; @@ -221,6 +238,7 @@ public final class MobileDataDownloadBuilder { } /** Set the Delta file decoder. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setDeltaDecoderOptional( Optional<DeltaDecoder> deltaDecoderOptional) { this.deltaDecoderOptional = deltaDecoderOptional; @@ -235,6 +253,7 @@ public final class MobileDataDownloadBuilder { * shared as an optimization. Please talk to <internal>@ on how to setup a shared Foreground * Download Service. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setForegroundDownloadServiceOptional( Optional<Class<?>> foregroundDownloadServiceClass) { this.foregroundDownloadServiceClassOptional = foregroundDownloadServiceClass; @@ -245,6 +264,7 @@ public final class MobileDataDownloadBuilder { * Sets the AccountSource that's used to wipeout account-related data at maintenance time. If this * method is not called, an account source based on AccountManager will be injected. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setAccountSourceOptional( Optional<AccountSource> accountSourceOptional) { this.accountSourceOptional = accountSourceOptional; @@ -252,6 +272,7 @@ public final class MobileDataDownloadBuilder { return this; } + @CanIgnoreReturnValue public MobileDataDownloadBuilder setCustomFileGroupValidatorOptional( Optional<CustomFileGroupValidator> customFileGroupValidatorOptional) { this.customFileGroupValidatorOptional = customFileGroupValidatorOptional; @@ -263,6 +284,7 @@ public final class MobileDataDownloadBuilder { * sources. If this is not called, experiment ids are not propagated. See <internal> for more * details. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setExperimentationConfigOptional( Optional<ExperimentationConfig> experimentationConfigOptional) { this.experimentationConfigOptional = experimentationConfigOptional; @@ -271,6 +293,7 @@ public final class MobileDataDownloadBuilder { // We use java.util.concurrent.Executor directly to create default Control Executor and // Download Executor. + public MobileDataDownload build() { Preconditions.checkNotNull(context); Preconditions.checkNotNull(taskSchedulerOptional); @@ -286,10 +309,10 @@ public final class MobileDataDownloadBuilder { // Submit commit task to sequentialControlExecutor to ensure that the commit task finishes // before any other API tasks can run. ListenableFuture<Void> commitFuture = - Futures.submitAsync( + PropagatedFutures.submitAsync( () -> configurator.get().commitToFlagSnapshot(), sequentialControlExecutor); - Futures.addCallback( + PropagatedFutures.addCallback( commitFuture, new FutureCallback<Void>() { @Override @@ -380,6 +403,7 @@ public final class MobileDataDownloadBuilder { foregroundDownloadServiceClassOptional, flags, singleFileDownloader, - customFileGroupValidatorOptional); + customFileGroupValidatorOptional, + component.getTimeSource()); } } diff --git a/java/com/google/android/libraries/mobiledatadownload/MobileDataDownloadImpl.java b/java/com/google/android/libraries/mobiledatadownload/MobileDataDownloadImpl.java index 4201b19..04abda1 100644 --- a/java/com/google/android/libraries/mobiledatadownload/MobileDataDownloadImpl.java +++ b/java/com/google/android/libraries/mobiledatadownload/MobileDataDownloadImpl.java @@ -15,16 +15,19 @@ */ package com.google.android.libraries.mobiledatadownload; -import static com.google.android.libraries.mobiledatadownload.tracing.TracePropagation.propagateAsyncCallable; import static com.google.android.libraries.mobiledatadownload.tracing.TracePropagation.propagateAsyncFunction; -import static com.google.android.libraries.mobiledatadownload.tracing.TracePropagation.propagateCallable; +import static com.google.android.libraries.mobiledatadownload.tracing.TracePropagation.propagateRunnable; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.util.concurrent.Futures.getDone; +import static com.google.common.util.concurrent.Futures.immediateFailedFuture; +import static com.google.common.util.concurrent.Futures.immediateFuture; +import static com.google.common.util.concurrent.Futures.immediateVoidFuture; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import android.accounts.Account; import android.content.Context; import android.net.Uri; import android.text.TextUtils; -import android.util.Pair; -import androidx.annotation.VisibleForTesting; import androidx.core.app.NotificationCompat; import androidx.core.app.NotificationManagerCompat; import com.google.android.libraries.mobiledatadownload.DownloadException.DownloadResultCode; @@ -32,24 +35,33 @@ import com.google.android.libraries.mobiledatadownload.TaskScheduler.ConstraintO import com.google.android.libraries.mobiledatadownload.TaskScheduler.NetworkState; import com.google.android.libraries.mobiledatadownload.account.AccountUtil; import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; +import com.google.android.libraries.mobiledatadownload.foreground.ForegroundDownloadKey; import com.google.android.libraries.mobiledatadownload.foreground.NotificationUtil; +import com.google.android.libraries.mobiledatadownload.internal.DownloadGroupState; +import com.google.android.libraries.mobiledatadownload.internal.ExceptionToMddResultMapper; +import com.google.android.libraries.mobiledatadownload.internal.MddConstants; import com.google.android.libraries.mobiledatadownload.internal.MobileDataDownloadManager; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupPair; import com.google.android.libraries.mobiledatadownload.internal.logging.EventLogger; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; +import com.google.android.libraries.mobiledatadownload.internal.util.DownloadFutureMap; import com.google.android.libraries.mobiledatadownload.internal.util.MddLiteConversionUtil; import com.google.android.libraries.mobiledatadownload.internal.util.ProtoConversionUtil; import com.google.android.libraries.mobiledatadownload.lite.Downloader; import com.google.android.libraries.mobiledatadownload.monitor.DownloadProgressMonitor; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedExecutionSequencer; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentFuture; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.AsyncFunction; -import com.google.common.util.concurrent.ExecutionSequencer; -import com.google.common.util.concurrent.FluentFuture; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListenableFutureTask; import com.google.mobiledatadownload.ClientConfigProto.ClientFile; import com.google.mobiledatadownload.ClientConfigProto.ClientFileGroup; import com.google.mobiledatadownload.DownloadConfigProto; @@ -57,15 +69,15 @@ import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; import com.google.mobiledatadownload.internal.MetadataProto.DataFile; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions; +import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions.DeviceNetworkPolicy; import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; import com.google.protobuf.Any; -import com.google.protobuf.GeneratedMessageLite; import com.google.protobuf.InvalidProtocolBufferException; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintWriter; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; @@ -73,11 +85,13 @@ import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import javax.annotation.Nullable; + /** * Default implementation for {@link * com.google.android.libraries.mobiledatadownload.MobileDataDownload}. */ class MobileDataDownloadImpl implements MobileDataDownload { + private static final String TAG = "MobileDataDownload"; private static final long DUMP_DEBUG_INFO_TIMEOUT = 3; @@ -90,6 +104,13 @@ class MobileDataDownloadImpl implements MobileDataDownload { private final Flags flags; private final Downloader singleFileDownloader; + // Track all the on-going foreground downloads. This map is keyed by ForegroundDownloadKey. + private final DownloadFutureMap<ClientFileGroup> foregroundDownloadFutureMap; + + // Track all on-going background download requests started by downloadFileGroup. This map is keyed + // by ForegroundDownloadKey so request can be kept in sync with foregroundDownloadFutureMap. + private final DownloadFutureMap<ClientFileGroup> downloadFutureMap; + // This executor will execute tasks sequentially. private final Executor sequentialControlExecutor; // ExecutionSequencer will execute a ListenableFuture and its Futures.transforms before taking the @@ -97,15 +118,12 @@ class MobileDataDownloadImpl implements MobileDataDownload { // ExecutionSequencer to guarantee Metadata synchronization. Currently only downloadFileGroup and // handleTask APIs do not use ExecutionSequencer since their execution could take long time and // using ExecutionSequencer would block other APIs. - private final ExecutionSequencer futureSerializer = ExecutionSequencer.create(); + private final PropagatedExecutionSequencer futureSerializer = + PropagatedExecutionSequencer.create(); private final Optional<DownloadProgressMonitor> downloadMonitorOptional; private final Optional<Class<?>> foregroundDownloadServiceClassOptional; private final AsyncFunction<DataFileGroupInternal, Boolean> customFileGroupValidator; - - // Synchronization will be done through sequentialControlExecutor - // Keep all the on-going foreground downloads. - @VisibleForTesting - final Map<String, ListenableFuture<ClientFileGroup>> keyToListenableFuture = new HashMap<>(); + private final TimeSource timeSource; MobileDataDownloadImpl( Context context, @@ -119,7 +137,8 @@ class MobileDataDownloadImpl implements MobileDataDownload { Optional<Class<?>> foregroundDownloadServiceClassOptional, Flags flags, Downloader singleFileDownloader, - Optional<CustomFileGroupValidator> customValidatorOptional) { + Optional<CustomFileGroupValidator> customValidatorOptional, + TimeSource timeSource) { this.context = context; this.eventLogger = eventLogger; this.fileGroupPopulatorList = fileGroupPopulatorList; @@ -137,6 +156,12 @@ class MobileDataDownloadImpl implements MobileDataDownload { mobileDataDownloadManager, sequentialControlExecutor, fileStorage); + this.downloadFutureMap = DownloadFutureMap.create(sequentialControlExecutor); + this.foregroundDownloadFutureMap = + DownloadFutureMap.create( + sequentialControlExecutor, + createCallbacksForForegroundService(context, foregroundDownloadServiceClassOptional)); + this.timeSource = timeSource; } // Wraps the custom validator because the validation at a lower level of the stack where @@ -148,16 +173,17 @@ class MobileDataDownloadImpl implements MobileDataDownload { Executor executor, SynchronousFileStorage fileStorage) { if (!validatorOptional.isPresent()) { - return unused -> Futures.immediateFuture(true); + return unused -> immediateFuture(true); } return internalFileGroup -> - Futures.transformAsync( + PropagatedFutures.transformAsync( createClientFileGroup( internalFileGroup, /* account= */ null, ClientFileGroup.Status.PENDING_CUSTOM_VALIDATION, /* preserveZipDirectories= */ false, + /* verifyIsolatedStructure= */ true, mobileDataDownloadManager, executor, fileStorage), @@ -166,63 +192,168 @@ class MobileDataDownloadImpl implements MobileDataDownload { executor); } + /** + * Functional interface used as callback for logging file group stats. Used to create file group + * stats from the result of the future. + * + * @see attachMddApiLogging + */ + private interface StatsFromApiResultCreator<T> { + DataDownloadFileGroupStats create(T result); + } + + /** + * Functional interface used as callback when logging API result. Used to get the API result code + * from the result of the API future if it succeeds. + * + * <p>Note: The need for this is due to {@link addFileGroup} returning false instead of an + * exception if it fails. For other APIs with proper exception handling, it should suffice to + * immediately return the success code. + * + * <p>TODO(b/143572409): Remove once addGroupForDownload is updated to return void. + * + * @see attachMddApiLogging + */ + private interface ResultCodeFromApiResultGetter<T> { + int get(T result); + } + + /** + * Helper function used to log mdd api stats. Adds FutureCallback to the {@code resultFuture} + * which is the result of mdd api call and logs in onSuccess and onFailure functions of callback. + * + * @param apiName Code of the api being logged. + * @param resultFuture Future result of the api call. + * @param startTimeNs start time in ns. + * @param defaultFileGroupStats Initial file group stats. + * @param statsCreator This functional interface is invoked from the onSuccess of FutureCallback + * with the result of the future. File group stats returned here is merged with the initial + * stats and logged. + */ + private <T> void attachMddApiLogging( + int apiName, + ListenableFuture<T> resultFuture, + long startTimeNs, + DataDownloadFileGroupStats defaultFileGroupStats, + StatsFromApiResultCreator<T> statsCreator, + ResultCodeFromApiResultGetter<T> resultCodeGetter) { + // Using listener instead of transform since we need to log even if the future fails. + // Note: Listener is being registered on directexecutor for accurate latency measurement. + resultFuture.addListener( + propagateRunnable( + () -> { + long latencyNs = timeSource.elapsedRealtimeNanos() - startTimeNs; + // Log the stats asynchronously. + // Note: To avoid adding latency to mdd api calls, log asynchronously. + var unused = + PropagatedFutures.submit( + () -> { + int resultCode; + T result = null; + DataDownloadFileGroupStats fileGroupStats = defaultFileGroupStats; + try { + result = Futures.getDone(resultFuture); + resultCode = resultCodeGetter.get(result); + } catch (Throwable t) { + resultCode = ExceptionToMddResultMapper.map(t); + } + + // Merge stats created from result of api with the default stats. + if (result != null) { + fileGroupStats = + fileGroupStats.toBuilder() + .mergeFrom(statsCreator.create(result)) + .build(); + } + + Void resultLog = null; + + eventLogger.logMddLibApiResultLog(resultLog); + }, + sequentialControlExecutor); + }), + directExecutor()); + } + @Override public ListenableFuture<Boolean> addFileGroup(AddFileGroupRequest addFileGroupRequest) { - return futureSerializer.submitAsync( - propagateAsyncCallable( - () -> { - LogUtil.d( - "%s: Adding for download group = '%s', variant = '%s' and associating it with" - + " account = '%s', variant = '%s'", - TAG, - addFileGroupRequest.dataFileGroup().getGroupName(), - addFileGroupRequest.dataFileGroup().getVariantId(), - String.valueOf(addFileGroupRequest.accountOptional().orNull()), - String.valueOf(addFileGroupRequest.variantIdOptional().orNull())); - - DataFileGroup dataFileGroup = addFileGroupRequest.dataFileGroup(); - - // Ensure that the owner package is always set as the host app. - if (!dataFileGroup.hasOwnerPackage()) { - dataFileGroup = - dataFileGroup.toBuilder().setOwnerPackage(context.getPackageName()).build(); - } else if (!context.getPackageName().equals(dataFileGroup.getOwnerPackage())) { - LogUtil.e( - "%s: Added group = '%s' with wrong owner package: '%s' v.s. '%s' ", - TAG, - dataFileGroup.getGroupName(), - context.getPackageName(), - dataFileGroup.getOwnerPackage()); - return Futures.immediateFuture(false); - } + long startTimeNs = timeSource.elapsedRealtimeNanos(); + + ListenableFuture<Boolean> resultFuture = + futureSerializer.submitAsync( + () -> addFileGroupHelper(addFileGroupRequest), sequentialControlExecutor); + + DataDownloadFileGroupStats defaultFileGroupStats = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(addFileGroupRequest.dataFileGroup().getGroupName()) + .setBuildId(addFileGroupRequest.dataFileGroup().getBuildId()) + .setVariantId(addFileGroupRequest.dataFileGroup().getVariantId()) + .setHasAccount(addFileGroupRequest.accountOptional().isPresent()) + .setFileGroupVersionNumber( + addFileGroupRequest.dataFileGroup().getFileGroupVersionNumber()) + .setOwnerPackage(addFileGroupRequest.dataFileGroup().getOwnerPackage()) + .setFileCount(addFileGroupRequest.dataFileGroup().getFileCount()) + .build(); + attachMddApiLogging( + 0, + resultFuture, + startTimeNs, + defaultFileGroupStats, + /* statsCreator= */ unused -> defaultFileGroupStats, + /* resultCodeGetter= */ succeeded -> succeeded ? 0 : 0); + + return resultFuture; + } - GroupKey.Builder groupKeyBuilder = - GroupKey.newBuilder() - .setGroupName(dataFileGroup.getGroupName()) - .setOwnerPackage(dataFileGroup.getOwnerPackage()); + private ListenableFuture<Boolean> addFileGroupHelper(AddFileGroupRequest addFileGroupRequest) { + LogUtil.d( + "%s: Adding for download group = '%s', variant = '%s', buildId = '%d' and" + + " associating it with account = '%s', variant = '%s'", + TAG, + addFileGroupRequest.dataFileGroup().getGroupName(), + addFileGroupRequest.dataFileGroup().getVariantId(), + addFileGroupRequest.dataFileGroup().getBuildId(), + String.valueOf(addFileGroupRequest.accountOptional().orNull()), + String.valueOf(addFileGroupRequest.variantIdOptional().orNull())); + + DataFileGroup dataFileGroup = addFileGroupRequest.dataFileGroup(); + + // Ensure that the owner package is always set as the host app. + if (!dataFileGroup.hasOwnerPackage()) { + dataFileGroup = dataFileGroup.toBuilder().setOwnerPackage(context.getPackageName()).build(); + } else if (!context.getPackageName().equals(dataFileGroup.getOwnerPackage())) { + LogUtil.e( + "%s: Added group = '%s' with wrong owner package: '%s' v.s. '%s' ", + TAG, + dataFileGroup.getGroupName(), + context.getPackageName(), + dataFileGroup.getOwnerPackage()); + return immediateFuture(false); + } - if (addFileGroupRequest.accountOptional().isPresent()) { - groupKeyBuilder.setAccount( - AccountUtil.serialize(addFileGroupRequest.accountOptional().get())); - } + GroupKey.Builder groupKeyBuilder = + GroupKey.newBuilder() + .setGroupName(dataFileGroup.getGroupName()) + .setOwnerPackage(dataFileGroup.getOwnerPackage()); - if (addFileGroupRequest.variantIdOptional().isPresent()) { - groupKeyBuilder.setVariantId(addFileGroupRequest.variantIdOptional().get()); - } + if (addFileGroupRequest.accountOptional().isPresent()) { + groupKeyBuilder.setAccount( + AccountUtil.serialize(addFileGroupRequest.accountOptional().get())); + } - try { - DataFileGroupInternal dataFileGroupInternal = - ProtoConversionUtil.convert(dataFileGroup); - return mobileDataDownloadManager.addGroupForDownloadInternal( - groupKeyBuilder.build(), dataFileGroupInternal, customFileGroupValidator); - } catch (InvalidProtocolBufferException e) { - // TODO(b/118137672): Consider rethrow exception instead of returning false. - LogUtil.e( - e, "%s: Unable to convert from DataFileGroup to DataFileGroupInternal.", TAG); - return Futures.immediateFuture(false); - } - }), - sequentialControlExecutor); + if (addFileGroupRequest.variantIdOptional().isPresent()) { + groupKeyBuilder.setVariantId(addFileGroupRequest.variantIdOptional().get()); + } + + try { + DataFileGroupInternal dataFileGroupInternal = ProtoConversionUtil.convert(dataFileGroup); + return mobileDataDownloadManager.addGroupForDownloadInternal( + groupKeyBuilder.build(), dataFileGroupInternal, customFileGroupValidator); + } catch (InvalidProtocolBufferException e) { + // TODO(b/118137672): Consider rethrow exception instead of returning false. + LogUtil.e(e, "%s: Unable to convert from DataFileGroup to DataFileGroupInternal.", TAG); + return immediateFuture(false); + } } // TODO: Change to return ListenableFuture<Void>. @@ -243,7 +374,7 @@ class MobileDataDownloadImpl implements MobileDataDownload { } GroupKey groupKey = groupKeyBuilder.build(); - return Futures.transform( + return PropagatedFutures.transform( mobileDataDownloadManager.removeFileGroup( groupKey, removeFileGroupRequest.pendingOnly()), voidArg -> true, @@ -257,29 +388,28 @@ class MobileDataDownloadImpl implements MobileDataDownload { RemoveFileGroupsByFilterRequest removeFileGroupsByFilterRequest) { return futureSerializer.submitAsync( () -> - FluentFuture.from(mobileDataDownloadManager.getAllFreshGroups()) + PropagatedFluentFuture.from(mobileDataDownloadManager.getAllFreshGroups()) .transformAsync( - allFreshGroups -> { + allFreshGroupKeyAndGroups -> { ImmutableSet.Builder<GroupKey> groupKeysToRemoveBuilder = ImmutableSet.builder(); - for (Pair<GroupKey, DataFileGroupInternal> keyDataFileGroupPair : - allFreshGroups) { + for (GroupKeyAndGroup groupKeyAndGroup : allFreshGroupKeyAndGroups) { if (applyRemoveFileGroupsFilter( - removeFileGroupsByFilterRequest, keyDataFileGroupPair)) { + removeFileGroupsByFilterRequest, groupKeyAndGroup)) { // Remove downloaded status so pending/downloaded versions of the same // group are treated as one. groupKeysToRemoveBuilder.add( - keyDataFileGroupPair.first.toBuilder().clearDownloaded().build()); + groupKeyAndGroup.groupKey().toBuilder().clearDownloaded().build()); } } ImmutableSet<GroupKey> groupKeysToRemove = groupKeysToRemoveBuilder.build(); if (groupKeysToRemove.isEmpty()) { - return Futures.immediateFuture( + return immediateFuture( RemoveFileGroupsByFilterResponse.newBuilder() .setRemovedFileGroupsCount(0) .build()); } - return Futures.transform( + return PropagatedFutures.transform( mobileDataDownloadManager.removeFileGroups(groupKeysToRemove.asList()), unused -> RemoveFileGroupsByFilterResponse.newBuilder() @@ -294,67 +424,135 @@ class MobileDataDownloadImpl implements MobileDataDownload { // Perform filtering using options from RemoveFileGroupsByFilterRequest private static boolean applyRemoveFileGroupsFilter( RemoveFileGroupsByFilterRequest removeFileGroupsByFilterRequest, - Pair<GroupKey, DataFileGroupInternal> keyDataFileGroupPair) { + GroupKeyAndGroup groupKeyAndGroup) { // If request filters by account, ensure account is present and is equal Optional<Account> accountOptional = removeFileGroupsByFilterRequest.accountOptional(); - if (!accountOptional.isPresent() && keyDataFileGroupPair.first.hasAccount()) { + if (!accountOptional.isPresent() && groupKeyAndGroup.groupKey().hasAccount()) { // Account must explicitly be provided in order to remove account associated file groups. return false; } if (accountOptional.isPresent() && !AccountUtil.serialize(accountOptional.get()) - .equals(keyDataFileGroupPair.first.getAccount())) { + .equals(groupKeyAndGroup.groupKey().getAccount())) { return false; } return true; } + /** + * Helper function to create {@link DataDownloadFileGroupStats} object from {@link + * GetFileGroupRequest} for getFileGroup() logging. + * + * <p>Used when the matching file group is not found or a failure occurred. + * file_group_version_number and build_id are set to -1 by default. + */ + private DataDownloadFileGroupStats createFileGroupStatsFromGetFileGroupRequest( + GetFileGroupRequest getFileGroupRequest) { + DataDownloadFileGroupStats.Builder fileGroupStatsBuilder = + DataDownloadFileGroupStats.newBuilder(); + fileGroupStatsBuilder.setFileGroupName(getFileGroupRequest.groupName()); + if (getFileGroupRequest.variantIdOptional().isPresent()) { + fileGroupStatsBuilder.setVariantId(getFileGroupRequest.variantIdOptional().get()); + } + if (getFileGroupRequest.accountOptional().isPresent()) { + fileGroupStatsBuilder.setHasAccount(true); + } else { + fileGroupStatsBuilder.setHasAccount(false); + } + + fileGroupStatsBuilder.setFileGroupVersionNumber( + MddConstants.FILE_GROUP_NOT_FOUND_FILE_GROUP_VERSION_NUMBER); + fileGroupStatsBuilder.setBuildId(MddConstants.FILE_GROUP_NOT_FOUND_BUILD_ID); + + return fileGroupStatsBuilder.build(); + } + // TODO: Futures.immediateFuture(null) uses a different annotation for Nullable. @SuppressWarnings("nullness") @Override public ListenableFuture<ClientFileGroup> getFileGroup(GetFileGroupRequest getFileGroupRequest) { - return futureSerializer.submitAsync( - () -> { - GroupKey.Builder groupKeyBuilder = - GroupKey.newBuilder() - .setGroupName(getFileGroupRequest.groupName()) - .setOwnerPackage(context.getPackageName()); + long startTimeNs = timeSource.elapsedRealtimeNanos(); - if (getFileGroupRequest.accountOptional().isPresent()) { - groupKeyBuilder.setAccount( - AccountUtil.serialize(getFileGroupRequest.accountOptional().get())); - } + ListenableFuture<ClientFileGroup> resultFuture = + futureSerializer.submitAsync( + () -> { + GroupKey groupKey = + createGroupKey( + getFileGroupRequest.groupName(), + getFileGroupRequest.accountOptional(), + getFileGroupRequest.variantIdOptional()); + return PropagatedFutures.transformAsync( + mobileDataDownloadManager.getFileGroup(groupKey, /* downloaded= */ true), + dataFileGroup -> + createClientFileGroupAndLogQueryStats( + groupKey, + dataFileGroup, + /* downloaded= */ true, + getFileGroupRequest.preserveZipDirectories(), + getFileGroupRequest.verifyIsolatedStructure()), + sequentialControlExecutor); + }, + sequentialControlExecutor); - if (getFileGroupRequest.variantIdOptional().isPresent()) { - groupKeyBuilder.setVariantId(getFileGroupRequest.variantIdOptional().get()); - } + attachMddApiLogging( + 0, + resultFuture, + startTimeNs, + createFileGroupStatsFromGetFileGroupRequest(getFileGroupRequest), + /* statsCreator= */ result -> createFileGroupDetails(result), + /* resultCodeGetter= */ unused -> 0); + return resultFuture; + } - GroupKey groupKey = groupKeyBuilder.build(); - return Futures.transformAsync( - mobileDataDownloadManager.getFileGroup(groupKey, /*downloaded=*/ true), - dataFileGroup -> - createClientFileGroupAndLogQueryStats( - groupKey, - dataFileGroup, - /*downloaded=*/ true, - getFileGroupRequest.preserveZipDirectories()), + @SuppressWarnings("nullness") + @Override + public ListenableFuture<DataFileGroup> readDataFileGroup( + ReadDataFileGroupRequest readDataFileGroupRequest) { + return futureSerializer.submitAsync( + () -> { + GroupKey groupKey = + createGroupKey( + readDataFileGroupRequest.groupName(), + readDataFileGroupRequest.accountOptional(), + readDataFileGroupRequest.variantIdOptional()); + return PropagatedFutures.transformAsync( + mobileDataDownloadManager.getFileGroup(groupKey, /* downloaded= */ true), + internalFileGroup -> immediateFuture(ProtoConversionUtil.reverse(internalFileGroup)), sequentialControlExecutor); }, sequentialControlExecutor); } + private GroupKey createGroupKey( + String groupName, Optional<Account> accountOptional, Optional<String> variantOptional) { + GroupKey.Builder groupKeyBuilder = + GroupKey.newBuilder().setGroupName(groupName).setOwnerPackage(context.getPackageName()); + + if (accountOptional.isPresent()) { + groupKeyBuilder.setAccount(AccountUtil.serialize(accountOptional.get())); + } + + if (variantOptional.isPresent()) { + groupKeyBuilder.setVariantId(variantOptional.get()); + } + + return groupKeyBuilder.build(); + } + private ListenableFuture<ClientFileGroup> createClientFileGroupAndLogQueryStats( GroupKey groupKey, @Nullable DataFileGroupInternal dataFileGroup, boolean downloaded, - boolean preserveZipDirectories) { - return Futures.transform( + boolean preserveZipDirectories, + boolean verifyIsolatedStructure) { + return PropagatedFutures.transform( createClientFileGroup( dataFileGroup, groupKey.hasAccount() ? groupKey.getAccount() : null, downloaded ? ClientFileGroup.Status.DOWNLOADED : ClientFileGroup.Status.PENDING, preserveZipDirectories, + verifyIsolatedStructure, mobileDataDownloadManager, sequentialControlExecutor, fileStorage), @@ -373,90 +571,91 @@ class MobileDataDownloadImpl implements MobileDataDownload { @Nullable String account, ClientFileGroup.Status status, boolean preserveZipDirectories, + boolean verifyIsolatedStructure, MobileDataDownloadManager manager, Executor executor, SynchronousFileStorage fileStorage) { if (dataFileGroup == null) { - return Futures.immediateFuture(null); + return immediateFuture(null); } - ClientFileGroup.Builder clientFileGroupBuilderInit = + ClientFileGroup.Builder clientFileGroupBuilder = ClientFileGroup.newBuilder() .setGroupName(dataFileGroup.getGroupName()) .setOwnerPackage(dataFileGroup.getOwnerPackage()) .setVersionNumber(dataFileGroup.getFileGroupVersionNumber()) +// .setCustomProperty(dataFileGroup.getCustomProperty()) .setBuildId(dataFileGroup.getBuildId()) .setVariantId(dataFileGroup.getVariantId()) .setStatus(status) .addAllLocale(dataFileGroup.getLocaleList()); if (account != null) { - clientFileGroupBuilderInit.setAccount(account); + clientFileGroupBuilder.setAccount(account); } if (dataFileGroup.hasCustomMetadata()) { - clientFileGroupBuilderInit.setCustomMetadata(dataFileGroup.getCustomMetadata()); + clientFileGroupBuilder.setCustomMetadata(dataFileGroup.getCustomMetadata()); } - ListenableFuture<ClientFileGroup.Builder> clientFileGroupBuilderFuture = - Futures.immediateFuture(clientFileGroupBuilderInit); - for (DataFile dataFile : dataFileGroup.getFileList()) { - clientFileGroupBuilderFuture = - Futures.transformAsync( - clientFileGroupBuilderFuture, - clientFileGroupBuilder -> { - if (status == ClientFileGroup.Status.DOWNLOADED - || status == ClientFileGroup.Status.PENDING_CUSTOM_VALIDATION) { - return Futures.transformAsync( - manager.getDataFileUri(dataFile, dataFileGroup), - fileUri -> { - if (fileUri == null) { - return Futures.immediateFailedFuture( - DownloadException.builder() - .setDownloadResultCode( - DownloadResultCode.DOWNLOADED_FILE_NOT_FOUND_ERROR) - .setMessage("getDataFileUri() resolved to null") - .build()); - } - try { - if (!preserveZipDirectories && fileStorage.isDirectory(fileUri)) { - String rootPath = fileUri.getPath(); - if (rootPath != null) { - clientFileGroupBuilder.addAllFile( - listAllClientFilesOfDirectory(fileStorage, fileUri, rootPath)); - } - } else { - clientFileGroupBuilder.addFile( - createClientFile( - dataFile.getFileId(), - dataFile.getByteSize(), - dataFile.getDownloadedFileByteSize(), - fileUri.toString(), - dataFile.hasCustomMetadata() - ? dataFile.getCustomMetadata() - : null)); + List<DataFile> dataFiles = dataFileGroup.getFileList(); + ListenableFuture<Void> addOnDeviceUrisFuture = immediateVoidFuture(); + if (status == ClientFileGroup.Status.DOWNLOADED + || status == ClientFileGroup.Status.PENDING_CUSTOM_VALIDATION) { + addOnDeviceUrisFuture = + PropagatedFluentFuture.from( + manager.getDataFileUris(dataFileGroup, verifyIsolatedStructure)) + .transformAsync( + dataFileUriMap -> { + for (DataFile dataFile : dataFiles) { + if (!dataFileUriMap.containsKey(dataFile)) { + return immediateFailedFuture( + DownloadException.builder() + .setDownloadResultCode( + DownloadResultCode.DOWNLOADED_FILE_NOT_FOUND_ERROR) + .setMessage("getDataFileUris() resolved to null") + .build()); + } + Uri uri = dataFileUriMap.get(dataFile); + + try { + if (!preserveZipDirectories && fileStorage.isDirectory(uri)) { + String rootPath = uri.getPath(); + if (rootPath != null) { + clientFileGroupBuilder.addAllFile( + listAllClientFilesOfDirectory(fileStorage, uri, rootPath)); } - } catch (IOException e) { - LogUtil.e(e, "Failed to list files under directory:" + fileUri); + } else { + clientFileGroupBuilder.addFile( + createClientFile( + dataFile.getFileId(), + dataFile.getByteSize(), + dataFile.getDownloadedFileByteSize(), + uri.toString(), + dataFile.hasCustomMetadata() + ? dataFile.getCustomMetadata() + : null)); } - return Futures.immediateFuture(clientFileGroupBuilder); - }, - executor); - } else { - clientFileGroupBuilder.addFile( - createClientFile( - dataFile.getFileId(), - dataFile.getByteSize(), - dataFile.getDownloadedFileByteSize(), - /* uri = */ null, - dataFile.hasCustomMetadata() ? dataFile.getCustomMetadata() : null)); - return Futures.immediateFuture(clientFileGroupBuilder); - } - }, - executor); + } catch (IOException e) { + LogUtil.e(e, "Failed to list files under directory:" + uri); + } + } + return immediateVoidFuture(); + }, + executor); + } else { + for (DataFile dataFile : dataFiles) { + clientFileGroupBuilder.addFile( + createClientFile( + dataFile.getFileId(), + dataFile.getByteSize(), + dataFile.getDownloadedFileByteSize(), + /* uri= */ null, + dataFile.hasCustomMetadata() ? dataFile.getCustomMetadata() : null)); + } } - return FluentFuture.from(clientFileGroupBuilderFuture) - .transform(GeneratedMessageLite.Builder::build, executor) + return PropagatedFluentFuture.from(addOnDeviceUrisFuture) + .transform(unused -> clientFileGroupBuilder.build(), executor) .catching(DownloadException.class, exn -> null, executor); } @@ -510,28 +709,29 @@ class MobileDataDownloadImpl implements MobileDataDownload { GetFileGroupsByFilterRequest getFileGroupsByFilterRequest) { return futureSerializer.submitAsync( () -> - Futures.transformAsync( + PropagatedFutures.transformAsync( mobileDataDownloadManager.getAllFreshGroups(), - allFreshGroups -> { + allFreshGroupKeyAndGroups -> { ListenableFuture<ImmutableList.Builder<ClientFileGroup>> clientFileGroupsBuilderFuture = - Futures.immediateFuture(ImmutableList.<ClientFileGroup>builder()); - for (Pair<GroupKey, DataFileGroupInternal> keyDataFileGroupPair : - allFreshGroups) { + immediateFuture(ImmutableList.<ClientFileGroup>builder()); + for (GroupKeyAndGroup groupKeyAndGroup : allFreshGroupKeyAndGroups) { clientFileGroupsBuilderFuture = - Futures.transformAsync( + PropagatedFutures.transformAsync( clientFileGroupsBuilderFuture, clientFileGroupsBuilder -> { - GroupKey groupKey = keyDataFileGroupPair.first; - DataFileGroupInternal dataFileGroup = keyDataFileGroupPair.second; + GroupKey groupKey = groupKeyAndGroup.groupKey(); + DataFileGroupInternal dataFileGroup = + groupKeyAndGroup.dataFileGroup(); if (applyFilter( getFileGroupsByFilterRequest, groupKey, dataFileGroup)) { - return Futures.transform( + return PropagatedFutures.transform( createClientFileGroupAndLogQueryStats( groupKey, dataFileGroup, groupKey.getDownloaded(), - getFileGroupsByFilterRequest.preserveZipDirectories()), + getFileGroupsByFilterRequest.preserveZipDirectories(), + getFileGroupsByFilterRequest.verifyIsolatedStructure()), clientFileGroup -> { if (clientFileGroup != null) { clientFileGroupsBuilder.add(clientFileGroup); @@ -540,12 +740,12 @@ class MobileDataDownloadImpl implements MobileDataDownload { }, sequentialControlExecutor); } - return Futures.immediateFuture(clientFileGroupsBuilder); + return immediateFuture(clientFileGroupsBuilder); }, sequentialControlExecutor); } - return Futures.transform( + return PropagatedFutures.transform( clientFileGroupsBuilderFuture, ImmutableList.Builder::build, sequentialControlExecutor); @@ -585,11 +785,19 @@ class MobileDataDownloadImpl implements MobileDataDownload { } /** - * Creates {@link IcingDataDownloadFileGroupStats} from {@link ClientFileGroup} for remote logging + * Creates {@link DataDownloadFileGroupStats} from {@link ClientFileGroup} for remote logging * purposes. */ - private static Void createFileGroupDetails(ClientFileGroup clientFileGroup) { - return null; + private static DataDownloadFileGroupStats createFileGroupDetails( + ClientFileGroup clientFileGroup) { + return DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(clientFileGroup.getGroupName()) + .setOwnerPackage(clientFileGroup.getOwnerPackage()) + .setFileGroupVersionNumber(clientFileGroup.getVersionNumber()) + .setFileCount(clientFileGroup.getFileCount()) + .setVariantId(clientFileGroup.getVariantId()) + .setBuildId(clientFileGroup.getBuildId()) + .build(); } @Override @@ -633,6 +841,37 @@ class MobileDataDownloadImpl implements MobileDataDownload { @Override public ListenableFuture<ClientFileGroup> downloadFileGroup( DownloadFileGroupRequest downloadFileGroupRequest) { + // Submit the call to sequentialControlExecutor, but don't use futureSerializer. This will + // ensure that multiple calls are enqueued to the executor in a FIFO order, but these calls + // won't block each other when the download is in progress. + return PropagatedFutures.submitAsync( + () -> + PropagatedFutures.transformAsync( + // Check if requested file group has already been downloaded + getDownloadGroupState(downloadFileGroupRequest), + downloadGroupState -> { + switch (downloadGroupState.getKind()) { + case IN_PROGRESS_FUTURE: + // If the file group download is in progress, return that future immediately + return downloadGroupState.inProgressFuture(); + case DOWNLOADED_GROUP: + // If the file group is already downloaded, return that immediately. + return immediateFuture(downloadGroupState.downloadedGroup()); + case PENDING_GROUP: + return downloadPendingFileGroup(downloadFileGroupRequest); + } + throw new AssertionError( + String.format( + "received unsupported DownloadGroupState kind %s", + downloadGroupState.getKind())); + }, + sequentialControlExecutor), + sequentialControlExecutor); + } + + /** Helper method to download a group after it's determined to be pending. */ + private ListenableFuture<ClientFileGroup> downloadPendingFileGroup( + DownloadFileGroupRequest downloadFileGroupRequest) { String groupName = downloadFileGroupRequest.groupName(); GroupKey.Builder groupKeyBuilder = GroupKey.newBuilder().setGroupName(groupName).setOwnerPackage(context.getPackageName()); @@ -647,74 +886,107 @@ class MobileDataDownloadImpl implements MobileDataDownload { GroupKey groupKey = groupKeyBuilder.build(); - ListenableFuture<ClientFileGroup> downloadFuture = - Futures.submitAsync( - () -> { - if (downloadFileGroupRequest.listenerOptional().isPresent()) { - if (downloadMonitorOptional.isPresent()) { - downloadMonitorOptional - .get() - .addDownloadListener( - groupName, downloadFileGroupRequest.listenerOptional().get()); - } else { - return Futures.immediateFailedFuture( - DownloadException.builder() - .setDownloadResultCode( - DownloadResultCode.DOWNLOAD_MONITOR_NOT_PROVIDED_ERROR) - .setMessage( - "downloadFileGroup: DownloadListener is present but Download Monitor" - + " is not provided!") - .build()); - } - } + if (downloadFileGroupRequest.listenerOptional().isPresent()) { + if (downloadMonitorOptional.isPresent()) { + downloadMonitorOptional + .get() + .addDownloadListener(groupName, downloadFileGroupRequest.listenerOptional().get()); + } else { + return immediateFailedFuture( + DownloadException.builder() + .setDownloadResultCode(DownloadResultCode.DOWNLOAD_MONITOR_NOT_PROVIDED_ERROR) + .setMessage( + "downloadFileGroup: DownloadListener is present but Download Monitor" + + " is not provided!") + .build()); + } + } - Optional<DownloadConditions> downloadConditions = - downloadFileGroupRequest.downloadConditionsOptional().isPresent() - ? Optional.of( - ProtoConversionUtil.convert( - downloadFileGroupRequest.downloadConditionsOptional().get())) - : Optional.absent(); - ListenableFuture<DataFileGroupInternal> downloadFileGroupFuture = - mobileDataDownloadManager.downloadFileGroup( - groupKey, downloadConditions, customFileGroupValidator); - - return Futures.transformAsync( - downloadFileGroupFuture, - dataFileGroup -> { - return Futures.transform( - createClientFileGroup( - dataFileGroup, - downloadFileGroupRequest.accountOptional().isPresent() - ? AccountUtil.serialize( - downloadFileGroupRequest.accountOptional().get()) - : null, - ClientFileGroup.Status.DOWNLOADED, - downloadFileGroupRequest.preserveZipDirectories(), - mobileDataDownloadManager, - sequentialControlExecutor, - fileStorage), - Preconditions::checkNotNull, - sequentialControlExecutor); - }, - sequentialControlExecutor); - }, - sequentialControlExecutor); + Optional<DownloadConditions> downloadConditions; + try { + downloadConditions = + downloadFileGroupRequest.downloadConditionsOptional().isPresent() + ? Optional.of( + ProtoConversionUtil.convert( + downloadFileGroupRequest.downloadConditionsOptional().get())) + : Optional.absent(); + } catch (InvalidProtocolBufferException e) { + return immediateFailedFuture(e); + } + + // Get the key used for the download future map + ForegroundDownloadKey downloadKey = + ForegroundDownloadKey.ofFileGroup( + downloadFileGroupRequest.groupName(), + downloadFileGroupRequest.accountOptional(), + downloadFileGroupRequest.variantIdOptional()); + + // Create a ListenableFutureTask to delay starting the downloadFuture until we can add the + // future to our map. + ListenableFutureTask<Void> startTask = ListenableFutureTask.create(() -> null); + ListenableFuture<ClientFileGroup> downloadFuture = + PropagatedFluentFuture.from(startTask) + .transformAsync( + unused -> + mobileDataDownloadManager.downloadFileGroup( + groupKey, downloadConditions, customFileGroupValidator), + sequentialControlExecutor) + .transformAsync( + dataFileGroup -> + createClientFileGroup( + dataFileGroup, + downloadFileGroupRequest.accountOptional().isPresent() + ? AccountUtil.serialize( + downloadFileGroupRequest.accountOptional().get()) + : null, + ClientFileGroup.Status.DOWNLOADED, + downloadFileGroupRequest.preserveZipDirectories(), + downloadFileGroupRequest.verifyIsolatedStructure(), + mobileDataDownloadManager, + sequentialControlExecutor, + fileStorage), + sequentialControlExecutor) + .transform(Preconditions::checkNotNull, sequentialControlExecutor); + + // Get a handle on the download task so we can get the CFG during transforms + PropagatedFluentFuture<ClientFileGroup> downloadTaskFuture = + PropagatedFluentFuture.from(downloadFutureMap.add(downloadKey.toString(), downloadFuture)) + .transformAsync( + unused -> { + // Now that the download future is added, start the task and return the future + startTask.run(); + return downloadFuture; + }, + sequentialControlExecutor); ListenableFuture<ClientFileGroup> transformFuture = - Futures.transform( - downloadFuture, - clientFileGroup -> { - if (downloadFileGroupRequest.listenerOptional().isPresent()) { - downloadFileGroupRequest.listenerOptional().get().onComplete(clientFileGroup); - if (downloadMonitorOptional.isPresent()) { - downloadMonitorOptional.get().removeDownloadListener(groupName); - } - } - return clientFileGroup; - }, - sequentialControlExecutor); + downloadTaskFuture + .transformAsync( + unused -> downloadFutureMap.remove(downloadKey.toString()), + sequentialControlExecutor) + .transformAsync( + unused -> { + ClientFileGroup clientFileGroup = getDone(downloadTaskFuture); + + if (downloadFileGroupRequest.listenerOptional().isPresent()) { + try { + downloadFileGroupRequest.listenerOptional().get().onComplete(clientFileGroup); + } catch (Exception e) { + LogUtil.w( + e, + "%s: Listener onComplete failed for group %s", + TAG, + clientFileGroup.getGroupName()); + } + if (downloadMonitorOptional.isPresent()) { + downloadMonitorOptional.get().removeDownloadListener(groupName); + } + } + return immediateFuture(clientFileGroup); + }, + sequentialControlExecutor); - Futures.addCallback( + PropagatedFutures.addCallback( transformFuture, new FutureCallback<ClientFileGroup>() { @Override @@ -722,10 +994,16 @@ class MobileDataDownloadImpl implements MobileDataDownload { @Override public void onFailure(Throwable t) { - if (downloadFileGroupRequest.listenerOptional().isPresent() - && downloadMonitorOptional.isPresent()) { - downloadMonitorOptional.get().removeDownloadListener(groupName); + if (downloadFileGroupRequest.listenerOptional().isPresent()) { + downloadFileGroupRequest.listenerOptional().get().onFailure(t); + + if (downloadMonitorOptional.isPresent()) { + downloadMonitorOptional.get().removeDownloadListener(groupName); + } } + + // Remove future from map + ListenableFuture<Void> unused = downloadFutureMap.remove(downloadKey.toString()); } }, sequentialControlExecutor); @@ -745,14 +1023,14 @@ class MobileDataDownloadImpl implements MobileDataDownload { DownloadFileGroupRequest downloadFileGroupRequest) { LogUtil.d("%s: downloadFileGroupWithForegroundService start.", TAG); if (!foregroundDownloadServiceClassOptional.isPresent()) { - return Futures.immediateFailedFuture( + return immediateFailedFuture( new IllegalArgumentException( "downloadFileGroupWithForegroundService: ForegroundDownloadService is not" + " provided!")); } if (!downloadMonitorOptional.isPresent()) { - return Futures.immediateFailedFuture( + return immediateFailedFuture( DownloadException.builder() .setDownloadResultCode(DownloadResultCode.DOWNLOAD_MONITOR_NOT_PROVIDED_ERROR) .setMessage( @@ -760,6 +1038,41 @@ class MobileDataDownloadImpl implements MobileDataDownload { .build()); } + // Submit the call to sequentialControlExecutor, but don't use futureSerializer. This will + // ensure that multiple calls are enqueued to the executor in a FIFO order, but these calls + // won't block each other when the download is in progress. + return PropagatedFutures.submitAsync( + () -> + PropagatedFutures.transformAsync( + // Check if requested file group has already been downloaded + getDownloadGroupState(downloadFileGroupRequest), + downloadGroupState -> { + switch (downloadGroupState.getKind()) { + case IN_PROGRESS_FUTURE: + // If the file group download is in progress, return that future immediately + return downloadGroupState.inProgressFuture(); + case DOWNLOADED_GROUP: + // If the file group is already downloaded, return that immediately + return immediateFuture(downloadGroupState.downloadedGroup()); + case PENDING_GROUP: + return downloadPendingFileGroupWithForegroundService( + downloadFileGroupRequest, downloadGroupState.pendingGroup()); + } + throw new AssertionError( + String.format( + "received unsupported DownloadGroupState kind %s", + downloadGroupState.getKind())); + }, + sequentialControlExecutor), + sequentialControlExecutor); + } + + /** + * Helper method to download a file group in the foreground after it has been confirmed to be + * pending. + */ + private ListenableFuture<ClientFileGroup> downloadPendingFileGroupWithForegroundService( + DownloadFileGroupRequest downloadFileGroupRequest, DataFileGroupInternal pendingGroup) { // It's OK to recreate the NotificationChannel since it can also be used to restore a // deleted channel and to update an existing channel's name, description, group, and/or // importance. @@ -778,106 +1091,109 @@ class MobileDataDownloadImpl implements MobileDataDownload { } GroupKey groupKey = groupKeyBuilder.build(); + ForegroundDownloadKey foregroundDownloadKey = + ForegroundDownloadKey.ofFileGroup( + groupName, + downloadFileGroupRequest.accountOptional(), + downloadFileGroupRequest.variantIdOptional()); + + DownloadListener downloadListenerWithNotification = + createDownloadListenerWithNotification(downloadFileGroupRequest, pendingGroup); + // The downloadMonitor will trigger the DownloadListener. + downloadMonitorOptional + .get() + .addDownloadListener( + downloadFileGroupRequest.groupName(), downloadListenerWithNotification); + + Optional<DownloadConditions> downloadConditions; + try { + downloadConditions = + downloadFileGroupRequest.downloadConditionsOptional().isPresent() + ? Optional.of( + ProtoConversionUtil.convert( + downloadFileGroupRequest.downloadConditionsOptional().get())) + : Optional.absent(); + } catch (InvalidProtocolBufferException e) { + return immediateFailedFuture(e); + } - ListenableFuture<ClientFileGroup> downloadFuture = - Futures.transformAsync( - // Check if requested file group has already been downloaded - tryToGetDownloadedFileGroup(downloadFileGroupRequest), - downloadedFileGroupOptional -> { - // If the file group has already been downloaded, return that one. - if (downloadedFileGroupOptional.isPresent()) { - return Futures.immediateFuture(downloadedFileGroupOptional.get()); - } - - // if there is the same on-going request, return that one. - if (keyToListenableFuture.containsKey(downloadFileGroupRequest.groupName())) { - // keyToListenableFuture.get must return Non-null since we check the containsKey - // above. - // checkNotNull is to suppress false alarm about @Nullable result. - return Preconditions.checkNotNull( - keyToListenableFuture.get(downloadFileGroupRequest.groupName())); - } - - // Only start the foreground download service when this is the first download - // request. - if (keyToListenableFuture.isEmpty()) { - NotificationUtil.startForegroundDownloadService( - context, - foregroundDownloadServiceClassOptional.get(), - downloadFileGroupRequest.groupName()); - } - - DownloadListener downloadListenerWithNotification = - createDownloadListenerWithNotification(downloadFileGroupRequest); - // The downloadMonitor will trigger the DownloadListener. - downloadMonitorOptional - .get() - .addDownloadListener( - downloadFileGroupRequest.groupName(), downloadListenerWithNotification); - - Optional<DownloadConditions> downloadConditions = - downloadFileGroupRequest.downloadConditionsOptional().isPresent() - ? Optional.of( - ProtoConversionUtil.convert( - downloadFileGroupRequest.downloadConditionsOptional().get())) - : Optional.absent(); - ListenableFuture<DataFileGroupInternal> downloadFileGroupFuture = - mobileDataDownloadManager.downloadFileGroup( - groupKey, downloadConditions, customFileGroupValidator); - - ListenableFuture<ClientFileGroup> transformFuture = - Futures.transformAsync( - downloadFileGroupFuture, - dataFileGroup -> { - return Futures.transform( - createClientFileGroup( - dataFileGroup, - downloadFileGroupRequest.accountOptional().isPresent() - ? AccountUtil.serialize( - downloadFileGroupRequest.accountOptional().get()) - : null, - ClientFileGroup.Status.DOWNLOADED, - downloadFileGroupRequest.preserveZipDirectories(), - mobileDataDownloadManager, - sequentialControlExecutor, - fileStorage), - Preconditions::checkNotNull, - sequentialControlExecutor); - }, - sequentialControlExecutor); - - Futures.addCallback( - transformFuture, - new FutureCallback<ClientFileGroup>() { - @Override - public void onSuccess(ClientFileGroup clientFileGroup) { - // Currently the MobStore monitor does not support onSuccess so we have to add - // callback to the download future here. - // TODO(b/148057674): Use the same logic as MDDLite to keep the foreground - // download service alive until the client's onComplete finishes. - downloadListenerWithNotification.onComplete(clientFileGroup); - } - - @Override - public void onFailure(Throwable t) { - // Currently the MobStore monitor does not support onFailure so we have to add - // callback to the download future here. - downloadListenerWithNotification.onFailure(t); - } - }, - sequentialControlExecutor); + // Create a ListenableFutureTask to delay starting the downloadFuture until we can add the + // future to our map. + ListenableFutureTask<Void> startTask = ListenableFutureTask.create(() -> null); + PropagatedFluentFuture<ClientFileGroup> downloadFileGroupFuture = + PropagatedFluentFuture.from(startTask) + .transformAsync( + unused -> + mobileDataDownloadManager.downloadFileGroup( + groupKey, downloadConditions, customFileGroupValidator), + sequentialControlExecutor) + .transformAsync( + dataFileGroup -> + createClientFileGroup( + dataFileGroup, + downloadFileGroupRequest.accountOptional().isPresent() + ? AccountUtil.serialize( + downloadFileGroupRequest.accountOptional().get()) + : null, + ClientFileGroup.Status.DOWNLOADED, + downloadFileGroupRequest.preserveZipDirectories(), + downloadFileGroupRequest.verifyIsolatedStructure(), + mobileDataDownloadManager, + sequentialControlExecutor, + fileStorage), + sequentialControlExecutor) + .transform(Preconditions::checkNotNull, sequentialControlExecutor); - keyToListenableFuture.put(downloadFileGroupRequest.groupName(), transformFuture); - return transformFuture; + ListenableFuture<ClientFileGroup> transformFuture = + PropagatedFutures.transformAsync( + foregroundDownloadFutureMap.add( + foregroundDownloadKey.toString(), downloadFileGroupFuture), + unused -> { + // Now that the download future is added, start the task and return the future + startTask.run(); + return downloadFileGroupFuture; }, sequentialControlExecutor); - return downloadFuture; + PropagatedFutures.addCallback( + transformFuture, + new FutureCallback<ClientFileGroup>() { + @Override + public void onSuccess(ClientFileGroup clientFileGroup) { + // Currently the MobStore monitor does not support onSuccess so we have to add + // callback to the download future here. + try { + downloadListenerWithNotification.onComplete(clientFileGroup); + } catch (Exception e) { + LogUtil.w( + e, + "%s: Listener onComplete failed for group %s", + TAG, + clientFileGroup.getGroupName()); + } + } + + @Override + public void onFailure(Throwable t) { + // Currently the MobStore monitor does not support onFailure so we have to add + // callback to the download future here. + downloadListenerWithNotification.onFailure(t); + } + }, + sequentialControlExecutor); + + return transformFuture; } - /** Helper method to check if file group has been downloaded and return it early. */ - private ListenableFuture<Optional<ClientFileGroup>> tryToGetDownloadedFileGroup( + /** Helper method to return a {@link DownloadGroupState} for the given request. */ + private ListenableFuture<DownloadGroupState> getDownloadGroupState( DownloadFileGroupRequest downloadFileGroupRequest) { + ForegroundDownloadKey foregroundDownloadKey = + ForegroundDownloadKey.ofFileGroup( + downloadFileGroupRequest.groupName(), + downloadFileGroupRequest.accountOptional(), + downloadFileGroupRequest.variantIdOptional()); + String groupName = downloadFileGroupRequest.groupName(); GroupKey.Builder groupKeyBuilder = GroupKey.newBuilder().setGroupName(groupName).setOwnerPackage(context.getPackageName()); @@ -886,101 +1202,164 @@ class MobileDataDownloadImpl implements MobileDataDownload { groupKeyBuilder.setAccount( AccountUtil.serialize(downloadFileGroupRequest.accountOptional().get())); } + + if (downloadFileGroupRequest.variantIdOptional().isPresent()) { + groupKeyBuilder.setVariantId(downloadFileGroupRequest.variantIdOptional().get()); + } + boolean isDownloadListenerPresent = downloadFileGroupRequest.listenerOptional().isPresent(); GroupKey groupKey = groupKeyBuilder.build(); - // Get pending and downloaded versions to tell if we should return downloaded version early - ListenableFuture<Pair<DataFileGroupInternal, DataFileGroupInternal>> fileGroupVersionsFuture = - Futures.transformAsync( - mobileDataDownloadManager.getFileGroup(groupKey, /* downloaded = */ false), - pendingDataFileGroup -> - Futures.transform( - mobileDataDownloadManager.getFileGroup(groupKey, /* downloaded = */ true), - downloadedDataFileGroup -> - Pair.create(pendingDataFileGroup, downloadedDataFileGroup), - sequentialControlExecutor), - sequentialControlExecutor); - - return Futures.transformAsync( - fileGroupVersionsFuture, - fileGroupVersionsPair -> { - // if pending version is not null, return absent - if (fileGroupVersionsPair.first != null) { - return Futures.immediateFuture(Optional.absent()); - } - // If both groups are null, return group not found failure - if (fileGroupVersionsPair.second == null) { - // TODO(b/174808410): Add Logging - // file group is not pending nor downloaded -- return failure. - DownloadException failure = - DownloadException.builder() - .setDownloadResultCode(DownloadResultCode.GROUP_NOT_FOUND_ERROR) - .setMessage("Nothing to download for file group: " + groupKey.getGroupName()) - .build(); - if (isDownloadListenerPresent) { - downloadFileGroupRequest.listenerOptional().get().onFailure(failure); - } - return Futures.immediateFailedFuture(failure); - } + return futureSerializer.submitAsync( + () -> { + ListenableFuture<Optional<ListenableFuture<ClientFileGroup>>> + foregroundDownloadFutureOptional = + foregroundDownloadFutureMap.get(foregroundDownloadKey.toString()); + ListenableFuture<Optional<ListenableFuture<ClientFileGroup>>> + backgroundDownloadFutureOptional = + downloadFutureMap.get(foregroundDownloadKey.toString()); + + return PropagatedFutures.whenAllSucceed( + foregroundDownloadFutureOptional, backgroundDownloadFutureOptional) + .callAsync( + () -> { + if (getDone(foregroundDownloadFutureOptional).isPresent()) { + return immediateFuture( + DownloadGroupState.ofInProgressFuture( + getDone(foregroundDownloadFutureOptional).get())); + } else if (getDone(backgroundDownloadFutureOptional).isPresent()) { + return immediateFuture( + DownloadGroupState.ofInProgressFuture( + getDone(backgroundDownloadFutureOptional).get())); + } - DataFileGroupInternal downloadedDataFileGroup = fileGroupVersionsPair.second; + // Get pending and downloaded versions to tell if we should return downloaded + // version early + ListenableFuture<GroupPair> fileGroupVersionsFuture = + PropagatedFutures.transformAsync( + mobileDataDownloadManager.getFileGroup( + groupKey, /* downloaded= */ false), + pendingDataFileGroup -> + PropagatedFutures.transform( + mobileDataDownloadManager.getFileGroup( + groupKey, /* downloaded= */ true), + downloadedDataFileGroup -> + GroupPair.create( + pendingDataFileGroup, downloadedDataFileGroup), + sequentialControlExecutor), + sequentialControlExecutor); - // Notify download listener (if present) that file group has been downloaded. - if (isDownloadListenerPresent) { - downloadMonitorOptional - .get() - .addDownloadListener( - downloadFileGroupRequest.groupName(), - downloadFileGroupRequest.listenerOptional().get()); - } - FluentFuture<Optional<ClientFileGroup>> transformFuture = - FluentFuture.from( - createClientFileGroup( - downloadedDataFileGroup, - downloadFileGroupRequest.accountOptional().isPresent() - ? AccountUtil.serialize( - downloadFileGroupRequest.accountOptional().get()) - : null, - ClientFileGroup.Status.DOWNLOADED, - downloadFileGroupRequest.preserveZipDirectories(), - mobileDataDownloadManager, - sequentialControlExecutor, - fileStorage)) - .transform(Preconditions::checkNotNull, sequentialControlExecutor) - .transform( - clientFileGroup -> { - if (isDownloadListenerPresent) { - downloadFileGroupRequest - .listenerOptional() - .get() - .onComplete(clientFileGroup); - downloadMonitorOptional.get().removeDownloadListener(groupName); - } - return Optional.of(clientFileGroup); - }, - sequentialControlExecutor); - transformFuture.addCallback( - new FutureCallback<Optional<ClientFileGroup>>() { - @Override - public void onSuccess(Optional<ClientFileGroup> result) {} - - @Override - public void onFailure(Throwable t) { - if (isDownloadListenerPresent) { - downloadMonitorOptional.get().removeDownloadListener(groupName); - } - } - }, - sequentialControlExecutor); + return PropagatedFutures.transformAsync( + fileGroupVersionsFuture, + fileGroupVersionsPair -> { + // if pending version is not null, return pending version + if (fileGroupVersionsPair.pendingGroup() != null) { + return immediateFuture( + DownloadGroupState.ofPendingGroup( + checkNotNull(fileGroupVersionsPair.pendingGroup()))); + } + // If both groups are null, return group not found failure + if (fileGroupVersionsPair.downloadedGroup() == null) { + // TODO(b/174808410): Add Logging + // file group is not pending nor downloaded -- return failure. + DownloadException failure = + DownloadException.builder() + .setDownloadResultCode(DownloadResultCode.GROUP_NOT_FOUND_ERROR) + .setMessage( + "Nothing to download for file group: " + + groupKey.getGroupName()) + .build(); + if (isDownloadListenerPresent) { + downloadFileGroupRequest.listenerOptional().get().onFailure(failure); + } + return immediateFailedFuture(failure); + } - return transformFuture; + DataFileGroupInternal downloadedDataFileGroup = + checkNotNull(fileGroupVersionsPair.downloadedGroup()); + + // Notify download listener (if present) that file group has been + // downloaded. + if (isDownloadListenerPresent) { + downloadMonitorOptional + .get() + .addDownloadListener( + downloadFileGroupRequest.groupName(), + downloadFileGroupRequest.listenerOptional().get()); + } + PropagatedFluentFuture<ClientFileGroup> transformFuture = + PropagatedFluentFuture.from( + createClientFileGroup( + downloadedDataFileGroup, + downloadFileGroupRequest.accountOptional().isPresent() + ? AccountUtil.serialize( + downloadFileGroupRequest.accountOptional().get()) + : null, + ClientFileGroup.Status.DOWNLOADED, + downloadFileGroupRequest.preserveZipDirectories(), + downloadFileGroupRequest.verifyIsolatedStructure(), + mobileDataDownloadManager, + sequentialControlExecutor, + fileStorage)) + .transform(Preconditions::checkNotNull, sequentialControlExecutor) + .transform( + clientFileGroup -> { + if (isDownloadListenerPresent) { + try { + downloadFileGroupRequest + .listenerOptional() + .get() + .onComplete(clientFileGroup); + } catch (Exception e) { + LogUtil.w( + e, + "%s: Listener onComplete failed for group %s", + TAG, + clientFileGroup.getGroupName()); + } + downloadMonitorOptional + .get() + .removeDownloadListener(groupName); + } + return clientFileGroup; + }, + sequentialControlExecutor); + transformFuture.addCallback( + new FutureCallback<ClientFileGroup>() { + @Override + public void onSuccess(ClientFileGroup result) {} + + @Override + public void onFailure(Throwable t) { + if (isDownloadListenerPresent) { + downloadMonitorOptional.get().removeDownloadListener(groupName); + } + } + }, + sequentialControlExecutor); + + // Use directExecutor here since we are performing a trivial operation. + return transformFuture.transform( + DownloadGroupState::ofDownloadedGroup, directExecutor()); + }, + sequentialControlExecutor); + }, + sequentialControlExecutor); }, sequentialControlExecutor); } private DownloadListener createDownloadListenerWithNotification( - DownloadFileGroupRequest downloadRequest) { + DownloadFileGroupRequest downloadRequest, DataFileGroupInternal fileGroup) { + + String networkPausedMessage = getNetworkPausedMessage(downloadRequest, fileGroup); + NotificationManagerCompat notificationManager = NotificationManagerCompat.from(context); + ForegroundDownloadKey foregroundDownloadKey = + ForegroundDownloadKey.ofFileGroup( + downloadRequest.groupName(), + downloadRequest.accountOptional(), + downloadRequest.variantIdOptional()); NotificationCompat.Builder notification = NotificationUtil.createNotificationBuilder( @@ -994,7 +1373,7 @@ class MobileDataDownloadImpl implements MobileDataDownload { NotificationUtil.createCancelAction( context, foregroundDownloadServiceClassOptional.get(), - downloadRequest.groupName(), + foregroundDownloadKey.toString(), notification, notificationKey); @@ -1004,133 +1383,192 @@ class MobileDataDownloadImpl implements MobileDataDownload { return new DownloadListener() { @Override public void onProgress(long currentSize) { - sequentialControlExecutor.execute( - () -> { - // There can be a race condition, where onPausedForConnectivity can be called - // after onComplete or onFailure which removes the future and the notification. - if (keyToListenableFuture.containsKey(downloadRequest.groupName()) - && downloadRequest.showNotifications() - == DownloadFileGroupRequest.ShowNotifications.ALL) { - notification - .setCategory(NotificationCompat.CATEGORY_PROGRESS) - .setSmallIcon(android.R.drawable.stat_sys_download) - .setProgress( - downloadRequest.groupSizeBytes(), - (int) currentSize, - /* indeterminate = */ downloadRequest.groupSizeBytes() <= 0); - notificationManager.notify(notificationKey, notification.build()); - } - if (downloadRequest.listenerOptional().isPresent()) { - downloadRequest.listenerOptional().get().onProgress(currentSize); - } - }); + // TODO(b/229123693): return this future once DownloadListener has an async api. + // There can be a race condition, where onProgress can be called + // after onComplete or onFailure which removes the future and the notification. + // Check foregroundDownloadFutureMap first before updating notification. + ListenableFuture<?> unused = + PropagatedFutures.transformAsync( + foregroundDownloadFutureMap.containsKey(foregroundDownloadKey.toString()), + futureInProgress -> { + if (futureInProgress + && downloadRequest.showNotifications() + == DownloadFileGroupRequest.ShowNotifications.ALL) { + notification + .setCategory(NotificationCompat.CATEGORY_PROGRESS) + .setSmallIcon(android.R.drawable.stat_sys_download) + .setProgress( + downloadRequest.groupSizeBytes(), + (int) currentSize, + /* indeterminate= */ downloadRequest.groupSizeBytes() <= 0); + notificationManager.notify(notificationKey, notification.build()); + } + if (downloadRequest.listenerOptional().isPresent()) { + downloadRequest.listenerOptional().get().onProgress(currentSize); + } + return immediateVoidFuture(); + }, + sequentialControlExecutor); } @Override public void pausedForConnectivity() { - sequentialControlExecutor.execute( - () -> { - // There can be a race condition, where pausedForConnectivity can be called - // after onComplete or onFailure which removes the future and the notification. - if (keyToListenableFuture.containsKey(downloadRequest.groupName()) - && downloadRequest.showNotifications() - == DownloadFileGroupRequest.ShowNotifications.ALL) { - notification - .setCategory(NotificationCompat.CATEGORY_STATUS) - .setContentText(NotificationUtil.getDownloadPausedMessage(context)) - .setSmallIcon(android.R.drawable.stat_sys_download) - .setOngoing(true) - // hide progress bar. - .setProgress(0, 0, false); - notificationManager.notify(notificationKey, notification.build()); - } - - if (downloadRequest.listenerOptional().isPresent()) { - downloadRequest.listenerOptional().get().pausedForConnectivity(); - } - }); + // TODO(b/229123693): return this future once DownloadListener has an async api. + // There can be a race condition, where pausedForConnectivity can be called + // after onComplete or onFailure which removes the future and the notification. + // Check foregroundDownloadFutureMap first before updating notification. + ListenableFuture<?> unused = + PropagatedFutures.transformAsync( + foregroundDownloadFutureMap.containsKey(foregroundDownloadKey.toString()), + futureInProgress -> { + if (futureInProgress + && downloadRequest.showNotifications() + == DownloadFileGroupRequest.ShowNotifications.ALL) { + notification + .setCategory(NotificationCompat.CATEGORY_STATUS) + .setContentText(networkPausedMessage) + .setSmallIcon(android.R.drawable.stat_sys_download) + .setOngoing(true) + // hide progress bar. + .setProgress(0, 0, false); + notificationManager.notify(notificationKey, notification.build()); + } + if (downloadRequest.listenerOptional().isPresent()) { + downloadRequest.listenerOptional().get().pausedForConnectivity(); + } + return immediateVoidFuture(); + }, + sequentialControlExecutor); } @Override public void onComplete(ClientFileGroup clientFileGroup) { - sequentialControlExecutor.execute( - () -> { - // Clear the notification action. - if (downloadRequest.showNotifications() - == DownloadFileGroupRequest.ShowNotifications.ALL) { - notification.mActions.clear(); - - NotificationUtil.cancelNotificationForKey(context, downloadRequest.groupName()); - } + // TODO(b/229123693): return this future once DownloadListener has an async api. + ListenableFuture<?> unused = + PropagatedFutures.submitAsync( + () -> { + boolean onCompleteFailed = false; + if (downloadRequest.listenerOptional().isPresent()) { + try { + downloadRequest.listenerOptional().get().onComplete(clientFileGroup); + } catch (Exception e) { + LogUtil.w( + e, + "%s: Delegate onComplete failed for group %s, showing failure" + + " notification.", + TAG, + clientFileGroup.getGroupName()); + onCompleteFailed = true; + } + } - keyToListenableFuture.remove(downloadRequest.groupName()); - // If there is no other on-going foreground download, shutdown the - // ForegroundDownloadService - if (keyToListenableFuture.isEmpty()) { - NotificationUtil.stopForegroundDownloadService( - context, foregroundDownloadServiceClassOptional.get()); - } + // Clear the notification action. + if (downloadRequest.showNotifications() + == DownloadFileGroupRequest.ShowNotifications.ALL) { + notification.mActions.clear(); + + if (onCompleteFailed) { + // Show download failed in notification. + notification + .setCategory(NotificationCompat.CATEGORY_STATUS) + .setContentText(NotificationUtil.getDownloadFailedMessage(context)) + .setOngoing(false) + .setSmallIcon(android.R.drawable.stat_sys_warning) + // hide progress bar. + .setProgress(0, 0, false); + + notificationManager.notify(notificationKey, notification.build()); + } else { + NotificationUtil.cancelNotificationForKey( + context, downloadRequest.groupName()); + } + } - if (downloadRequest.listenerOptional().isPresent()) { - downloadRequest.listenerOptional().get().onComplete(clientFileGroup); - } + downloadMonitorOptional.get().removeDownloadListener(downloadRequest.groupName()); - downloadMonitorOptional.get().removeDownloadListener(downloadRequest.groupName()); - }); + return foregroundDownloadFutureMap.remove(foregroundDownloadKey.toString()); + }, + sequentialControlExecutor); } @Override public void onFailure(Throwable t) { - sequentialControlExecutor.execute( - () -> { - if (downloadRequest.showNotifications() - == DownloadFileGroupRequest.ShowNotifications.ALL) { - // Clear the notification action. - notification.mActions.clear(); - - // Show download failed in notification. - notification - .setCategory(NotificationCompat.CATEGORY_STATUS) - .setContentText(NotificationUtil.getDownloadFailedMessage(context)) - .setOngoing(false) - .setSmallIcon(android.R.drawable.stat_sys_warning) - // hide progress bar. - .setProgress(0, 0, false); - - notificationManager.notify(notificationKey, notification.build()); - } - keyToListenableFuture.remove(downloadRequest.groupName()); + // TODO(b/229123693): return this future once DownloadListener has an async api. + ListenableFuture<?> unused = + PropagatedFutures.submitAsync( + () -> { + if (downloadRequest.showNotifications() + == DownloadFileGroupRequest.ShowNotifications.ALL) { + // Clear the notification action. + notification.mActions.clear(); + + // Show download failed in notification. + notification + .setCategory(NotificationCompat.CATEGORY_STATUS) + .setContentText(NotificationUtil.getDownloadFailedMessage(context)) + .setOngoing(false) + .setSmallIcon(android.R.drawable.stat_sys_warning) + // hide progress bar. + .setProgress(0, 0, false); + + notificationManager.notify(notificationKey, notification.build()); + } - // If there is no other on-going foreground download, shutdown the - // ForegroundDownloadService - if (keyToListenableFuture.isEmpty()) { - NotificationUtil.stopForegroundDownloadService( - context, foregroundDownloadServiceClassOptional.get()); - } + if (downloadRequest.listenerOptional().isPresent()) { + downloadRequest.listenerOptional().get().onFailure(t); + } + downloadMonitorOptional.get().removeDownloadListener(downloadRequest.groupName()); - if (downloadRequest.listenerOptional().isPresent()) { - downloadRequest.listenerOptional().get().onFailure(t); - } - downloadMonitorOptional.get().removeDownloadListener(downloadRequest.groupName()); - }); + return foregroundDownloadFutureMap.remove(foregroundDownloadKey.toString()); + }, + sequentialControlExecutor); } }; } + // Helper method to get the correct network paused message + private String getNetworkPausedMessage( + DownloadFileGroupRequest downloadRequest, DataFileGroupInternal fileGroup) { + DeviceNetworkPolicy networkPolicyForDownload = + fileGroup.getDownloadConditions().getDeviceNetworkPolicy(); + if (downloadRequest.downloadConditionsOptional().isPresent()) { + try { + networkPolicyForDownload = + ProtoConversionUtil.convert(downloadRequest.downloadConditionsOptional().get()) + .getDeviceNetworkPolicy(); + } catch (InvalidProtocolBufferException unused) { + // Do nothing -- we will rely on the file group's network policy. + } + } + + switch (networkPolicyForDownload) { + case DOWNLOAD_FIRST_ON_WIFI_THEN_ON_ANY_NETWORK: // fallthrough + case DOWNLOAD_ONLY_ON_WIFI: + return NotificationUtil.getDownloadPausedWifiMessage(context); + default: + return NotificationUtil.getDownloadPausedMessage(context); + } + } + @Override public void cancelForegroundDownload(String downloadKey) { LogUtil.d("%s: CancelForegroundDownload for key = %s", TAG, downloadKey); - sequentialControlExecutor.execute( - () -> { - if (keyToListenableFuture.containsKey(downloadKey)) { - keyToListenableFuture.get(downloadKey).cancel(true); - } else { - // downloadKey is not a file group, attempt cancel with internal MDD Lite instance in - // case it's a single file uri (cancel call is a noop if internal MDD Lite doesn't know - // about it). - singleFileDownloader.cancelForegroundDownload(downloadKey); - } - }); + ListenableFuture<?> unused = + PropagatedFutures.transformAsync( + foregroundDownloadFutureMap.get(downloadKey), + downloadFuture -> { + if (downloadFuture.isPresent()) { + LogUtil.v( + "%s: CancelForegroundDownload future found for key = %s, cancelling...", + TAG, downloadKey); + downloadFuture.get().cancel(false); + } + return immediateVoidFuture(); + }, + sequentialControlExecutor); + // Attempt cancel with internal MDD Lite instance in case it's a single file uri (cancel call is + // a noop if internal MDD Lite doesn't know about it). + singleFileDownloader.cancelForegroundDownload(downloadKey); } @Override @@ -1141,11 +1579,10 @@ class MobileDataDownloadImpl implements MobileDataDownload { @Override public ListenableFuture<Void> schedulePeriodicBackgroundTasks() { return futureSerializer.submit( - propagateCallable( - () -> { - schedulePeriodicTasksInternal(/* constraintOverridesMap = */ Optional.absent()); - return null; - }), + () -> { + schedulePeriodicTasksInternal(/* constraintOverridesMap= */ Optional.absent()); + return null; + }, sequentialControlExecutor); } @@ -1153,11 +1590,10 @@ class MobileDataDownloadImpl implements MobileDataDownload { public ListenableFuture<Void> schedulePeriodicBackgroundTasks( Optional<Map<String, ConstraintOverrides>> constraintOverridesMap) { return futureSerializer.submit( - propagateCallable( - () -> { - schedulePeriodicTasksInternal(constraintOverridesMap); - return null; - }), + () -> { + schedulePeriodicTasksInternal(constraintOverridesMap); + return null; + }, sequentialControlExecutor); } @@ -1211,6 +1647,30 @@ class MobileDataDownloadImpl implements MobileDataDownload { } @Override + public ListenableFuture<Void> cancelPeriodicBackgroundTasks() { + return futureSerializer.submit( + () -> { + cancelPeriodicTasksInternal(); + return null; + }, + sequentialControlExecutor); + } + + private void cancelPeriodicTasksInternal() { + if (!taskSchedulerOptional.isPresent()) { + LogUtil.w("%s: Called cancelPeriodicTasksInternal when taskScheduler is not provided.", TAG); + return; + } + + TaskScheduler taskScheduler = taskSchedulerOptional.get(); + + taskScheduler.cancelPeriodicTask(TaskScheduler.CHARGING_PERIODIC_TASK); + taskScheduler.cancelPeriodicTask(TaskScheduler.MAINTENANCE_PERIODIC_TASK); + taskScheduler.cancelPeriodicTask(TaskScheduler.CELLULAR_CHARGING_PERIODIC_TASK); + taskScheduler.cancelPeriodicTask(TaskScheduler.WIFI_CHARGING_PERIODIC_TASK); + } + + @Override public ListenableFuture<Void> handleTask(String tag) { // All work done here that touches metadata (MobileDataDownloadManager) should be serialized // through sequentialControlExecutor. @@ -1221,7 +1681,7 @@ class MobileDataDownloadImpl implements MobileDataDownload { case TaskScheduler.CHARGING_PERIODIC_TASK: ListenableFuture<Void> refreshFileGroupsFuture = refreshFileGroups(); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( refreshFileGroupsFuture, propagateAsyncFunction( v -> mobileDataDownloadManager.verifyAllPendingGroups(customFileGroupValidator)), @@ -1235,7 +1695,7 @@ class MobileDataDownloadImpl implements MobileDataDownload { default: LogUtil.d("%s: gcm task doesn't belong to MDD", TAG); - return Futures.immediateFailedFuture( + return immediateFailedFuture( new IllegalArgumentException("Unknown task tag sent to MDD.handleTask() " + tag)); } } @@ -1243,7 +1703,7 @@ class MobileDataDownloadImpl implements MobileDataDownload { private ListenableFuture<Void> refreshAndDownload(boolean onWifi) { // We will do 2 passes to support 2-step downloads. In each step, we will refresh and then // download. - return FluentFuture.from(refreshFileGroups()) + return PropagatedFluentFuture.from(refreshFileGroups()) .transformAsync( v -> mobileDataDownloadManager.downloadAllPendingGroups( @@ -1263,7 +1723,8 @@ class MobileDataDownloadImpl implements MobileDataDownload { refreshFutures.add(fileGroupPopulator.refreshFileGroups(this)); } - return Futures.whenAllComplete(refreshFutures).call(() -> null, sequentialControlExecutor); + return PropagatedFutures.whenAllComplete(refreshFutures) + .call(() -> null, sequentialControlExecutor); } @Override @@ -1272,6 +1733,12 @@ class MobileDataDownloadImpl implements MobileDataDownload { } @Override + public ListenableFuture<Void> collectGarbage() { + return futureSerializer.submitAsync( + mobileDataDownloadManager::removeExpiredGroupsAndFiles, sequentialControlExecutor); + } + + @Override public ListenableFuture<Void> clear() { return futureSerializer.submitAsync( mobileDataDownloadManager::clear, sequentialControlExecutor); @@ -1307,6 +1774,29 @@ class MobileDataDownloadImpl implements MobileDataDownload { public ListenableFuture<Void> reportUsage(UsageEvent usageEvent) { eventLogger.logMddUsageEvent(createFileGroupDetails(usageEvent.clientFileGroup()), null); - return Futures.immediateVoidFuture(); + return immediateVoidFuture(); + } + + private static DownloadFutureMap.StateChangeCallbacks createCallbacksForForegroundService( + Context context, Optional<Class<?>> foregroundDownloadServiceClassOptional) { + return new DownloadFutureMap.StateChangeCallbacks() { + @Override + public void onAdd(String key, int newSize) { + // Only start foreground service if this is the first future we are adding. + if (newSize == 1 && foregroundDownloadServiceClassOptional.isPresent()) { + NotificationUtil.startForegroundDownloadService( + context, foregroundDownloadServiceClassOptional.get(), key); + } + } + + @Override + public void onRemove(String key, int newSize) { + // Only stop foreground service if there are no more futures remaining. + if (newSize == 0 && foregroundDownloadServiceClassOptional.isPresent()) { + NotificationUtil.stopForegroundDownloadService( + context, foregroundDownloadServiceClassOptional.get(), key); + } + } + }; } } diff --git a/java/com/google/android/libraries/mobiledatadownload/ReadDataFileGroupRequest.java b/java/com/google/android/libraries/mobiledatadownload/ReadDataFileGroupRequest.java new file mode 100644 index 0000000..88fc970 --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/ReadDataFileGroupRequest.java @@ -0,0 +1,54 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload; + +import android.accounts.Account; +import com.google.auto.value.AutoValue; +import com.google.common.base.Optional; +import javax.annotation.concurrent.Immutable; + +/** Request to get a single file group definition. */ +@AutoValue +@Immutable +public abstract class ReadDataFileGroupRequest { + + public abstract String groupName(); + + public abstract Optional<Account> accountOptional(); + + public abstract Optional<String> variantIdOptional(); + + public static Builder newBuilder() { + return new AutoValue_ReadDataFileGroupRequest.Builder(); + } + + /** Builder for {@link ReadDataFileGroupRequest}. */ + @AutoValue.Builder + public abstract static class Builder { + Builder() {} + + /** Sets the data file group, which is required. */ + public abstract Builder setGroupName(String groupName); + + /** Sets the account associated with the group, which is optional. */ + public abstract Builder setAccountOptional(Optional<Account> accountOptional); + + /** Sets the variant id associated with the group, which is optional. */ + public abstract Builder setVariantIdOptional(Optional<String> variantIdOptional); + + public abstract ReadDataFileGroupRequest build(); + } +} diff --git a/java/com/google/android/libraries/mobiledatadownload/TaskScheduler.java b/java/com/google/android/libraries/mobiledatadownload/TaskScheduler.java index c06c22b..dc6147c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/TaskScheduler.java +++ b/java/com/google/android/libraries/mobiledatadownload/TaskScheduler.java @@ -148,4 +148,14 @@ public interface TaskScheduler { // update all clients. schedulePeriodicTask(tag, period, networkState); } + + /** + * Cancel future invocations of a previously-scheduled task. No guarantee is made whether the task + * will be interrupted if it's currently running. + * + * @param tag tag of the scheduled task. + */ + default void cancelPeriodicTask(String tag) { + // TODO(b/223822302): remove default once all implementations have been updated to include it + } } diff --git a/java/com/google/android/libraries/mobiledatadownload/TimeSource.java b/java/com/google/android/libraries/mobiledatadownload/TimeSource.java index d632382..d045568 100644 --- a/java/com/google/android/libraries/mobiledatadownload/TimeSource.java +++ b/java/com/google/android/libraries/mobiledatadownload/TimeSource.java @@ -15,13 +15,11 @@ */ package com.google.android.libraries.mobiledatadownload; -/** - * Interface through which the SystemClock can be read. - * - * <p>This interface is analogous to {@code com.google.common.time.TimeSource#now#toEpochMilli} - * without the dependency on Java8. - */ +/** Interface through which the SystemClock can be read. */ public interface TimeSource { /** Returns the current system time in milliseconds since January 1, 1970 00:00:00 UTC. */ long currentTimeMillis(); + + /** Returns nanoseconds since boot, including time spent in sleep. */ + long elapsedRealtimeNanos(); } diff --git a/java/com/google/android/libraries/mobiledatadownload/account/AccountUtil.java b/java/com/google/android/libraries/mobiledatadownload/account/AccountUtil.java index 012226e..4c1ad8a 100644 --- a/java/com/google/android/libraries/mobiledatadownload/account/AccountUtil.java +++ b/java/com/google/android/libraries/mobiledatadownload/account/AccountUtil.java @@ -41,7 +41,11 @@ public final class AccountUtil { return new Account(name, type); } - /** Serializes an {@link Account} into a string. */ + /** + * Serializes an {@link Account} into a string. + * + * <p>TODO(b/222110940): make this function consistent with deserialize. + */ public static String serialize(Account account) { return account.type + ACCOUNT_DELIMITER + account.name; } @@ -49,10 +53,14 @@ public final class AccountUtil { /** * Deserializes a string into an {@link Account}. * - * @return The account parsed from string. Returns null if there is any error during parse. + * @return The account parsed from string. Returns null if the accountStr is empty or if there is + * any error during parse. */ @Nullable public static Account deserialize(String accountStr) { + if (accountStr.isEmpty()) { + return null; + } int splitIndex = accountStr.indexOf(ACCOUNT_DELIMITER); if (splitIndex < 0) { LogUtil.e("%s: Unable to parse Account with string = '%s'", TAG, accountStr); diff --git a/java/com/google/android/libraries/mobiledatadownload/account/BUILD b/java/com/google/android/libraries/mobiledatadownload/account/BUILD index cd9bd61..23cd484 100644 --- a/java/com/google/android/libraries/mobiledatadownload/account/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/account/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], diff --git a/java/com/google/android/libraries/mobiledatadownload/annotations/BUILD b/java/com/google/android/libraries/mobiledatadownload/annotations/BUILD index 9bc3d32..6066dbe 100644 --- a/java/com/google/android/libraries/mobiledatadownload/annotations/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/annotations/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], diff --git a/java/com/google/android/libraries/mobiledatadownload/delta/BUILD b/java/com/google/android/libraries/mobiledatadownload/delta/BUILD index 50556e2..afd5b5c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/delta/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/delta/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/BUILD b/java/com/google/android/libraries/mobiledatadownload/downloader/BUILD index 3831c2e..95150b5 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/DownloadConstraints.java b/java/com/google/android/libraries/mobiledatadownload/downloader/DownloadConstraints.java index e6489c9..9801982 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/DownloadConstraints.java +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/DownloadConstraints.java @@ -17,6 +17,7 @@ package com.google.android.libraries.mobiledatadownload.downloader; import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableSet; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.util.EnumSet; import java.util.Set; @@ -107,6 +108,7 @@ public abstract class DownloadConstraints { abstract ImmutableSet.Builder<NetworkType> requiredNetworkTypesBuilder(); + @CanIgnoreReturnValue public final Builder addRequiredNetworkType(NetworkType networkType) { requiredNetworkTypesBuilder().add(networkType); return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/MultiSchemeFileDownloader.java b/java/com/google/android/libraries/mobiledatadownload/downloader/MultiSchemeFileDownloader.java index c0b82a3..7dfc5b4 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/MultiSchemeFileDownloader.java +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/MultiSchemeFileDownloader.java @@ -24,6 +24,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.errorprone.annotations.CheckReturnValue; import java.net.MalformedURLException; import java.util.HashMap; @@ -41,6 +42,7 @@ public final class MultiSchemeFileDownloader implements FileDownloader { private final Map<String, FileDownloader> schemeToDownloader = new HashMap<>(); /** Associates a url scheme (e.g. "http") with a specific {@link FileDownloader} delegate. */ + @CanIgnoreReturnValue public MultiSchemeFileDownloader.Builder addScheme(String scheme, FileDownloader downloader) { schemeToDownloader.put( Preconditions.checkNotNull(scheme), Preconditions.checkNotNull(downloader)); diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/inline/BUILD b/java/com/google/android/libraries/mobiledatadownload/downloader/inline/BUILD index a09dd65..5c4fa53 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/inline/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/inline/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -32,6 +33,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file/openers:stream", "//java/com/google/android/libraries/mobiledatadownload/internal:MddConstants", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "@com_google_guava_guava", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/inline/InlineFileDownloader.java b/java/com/google/android/libraries/mobiledatadownload/downloader/inline/InlineFileDownloader.java index 8f3d472..9102b56 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/inline/InlineFileDownloader.java +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/inline/InlineFileDownloader.java @@ -16,6 +16,8 @@ package com.google.android.libraries.mobiledatadownload.downloader.inline; import static com.google.android.libraries.mobiledatadownload.internal.MddConstants.INLINE_FILE_URL_SCHEME; +import static com.google.common.util.concurrent.Futures.immediateFailedFuture; +import static com.google.common.util.concurrent.Futures.immediateVoidFuture; import com.google.android.libraries.mobiledatadownload.DownloadException; import com.google.android.libraries.mobiledatadownload.DownloadException.DownloadResultCode; @@ -26,8 +28,8 @@ import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStora import com.google.android.libraries.mobiledatadownload.file.openers.ReadStreamOpener; import com.google.android.libraries.mobiledatadownload.file.openers.WriteStreamOpener; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.io.ByteStreams; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import java.io.IOException; import java.io.InputStream; @@ -67,7 +69,7 @@ public final class InlineFileDownloader implements FileDownloader { LogUtil.e( "%s: Invalid url given, expected to start with 'inlinefile:', but was %s", TAG, downloadRequest.urlToDownload()); - return Futures.immediateFailedFuture( + return immediateFailedFuture( DownloadException.builder() .setDownloadResultCode(DownloadResultCode.INVALID_INLINE_FILE_URL_SCHEME) .setMessage("InlineFileDownloader only supports copying inlinefile: scheme") @@ -78,7 +80,7 @@ public final class InlineFileDownloader implements FileDownloader { InlineDownloadParams inlineDownloadParams = downloadRequest.inlineDownloadParamsOptional().get(); - return Futures.submitAsync( + return PropagatedFutures.submitAsync( () -> { try (InputStream inlineFileStream = getInputStream(inlineDownloadParams); OutputStream destinationStream = @@ -87,13 +89,13 @@ public final class InlineFileDownloader implements FileDownloader { destinationStream.flush(); } catch (IOException e) { LogUtil.e(e, "%s: Unable to copy file content.", TAG); - return Futures.immediateFailedFuture( + return immediateFailedFuture( DownloadException.builder() .setCause(e) .setDownloadResultCode(DownloadResultCode.INLINE_FILE_IO_ERROR) .build()); } - return Futures.immediateVoidFuture(); + return immediateVoidFuture(); }, downloadExecutor); } diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/BUILD b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/BUILD index 4774127..5d8f6d6 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -34,6 +35,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file/integration/downloader:downloader2", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "@androidx_concurrent_concurrent", "@com_google_code_findbugs_jsr305", "@com_google_guava_guava", "@downloader", @@ -64,6 +66,7 @@ android_library( deps = [ "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", "@com_google_code_findbugs_jsr305", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/ExceptionHandler.java b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/ExceptionHandler.java index 759c805..b096bd6 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/ExceptionHandler.java +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/ExceptionHandler.java @@ -71,7 +71,7 @@ public final class ExceptionHandler { return (DownloadException) throwable; } - DownloadResultCode code = mapExceptionToDownloadResultCode(throwable, /* iteration = */ 0); + DownloadResultCode code = mapExceptionToDownloadResultCode(throwable, /* iteration= */ 0); return DownloadException.builder() .setMessage(message) diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/Offroad2FileDownloader.java b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/Offroad2FileDownloader.java index 682045f..b88e005 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/Offroad2FileDownloader.java +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/Offroad2FileDownloader.java @@ -17,6 +17,7 @@ package com.google.android.libraries.mobiledatadownload.downloader.offroad; import android.net.Uri; import android.util.Pair; + import com.google.android.downloader.DownloadConstraints; import com.google.android.downloader.DownloadConstraints.NetworkType; import com.google.android.downloader.DownloadDestination; @@ -41,9 +42,11 @@ import com.google.common.base.Strings; import com.google.common.util.concurrent.FluentFuture; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; + import java.io.IOException; import java.net.URI; import java.util.concurrent.Executor; + import javax.annotation.Nullable; /** @@ -51,162 +54,180 @@ import javax.annotation.Nullable; * com.google.android.libraries.mobiledatadownload.downloader.FileDownloader} using <internal> */ public final class Offroad2FileDownloader implements FileDownloader { - private static final String TAG = "Offroad2FileDownloader"; - - private final Downloader downloader; - private final SynchronousFileStorage fileStorage; - private final Executor downloadExecutor; - private final DownloadMetadataStore downloadMetadataStore; - private final ExceptionHandler exceptionHandler; - private final Optional<Integer> defaultTrafficTag; - @Nullable private final OAuthTokenProvider authTokenProvider; - - // TODO(b/208703042): refactor injection to remove dependency on ProtoDataStore - public Offroad2FileDownloader( - Downloader downloader, - SynchronousFileStorage fileStorage, - Executor downloadExecutor, - @Nullable OAuthTokenProvider authTokenProvider, - DownloadMetadataStore downloadMetadataStore, - ExceptionHandler exceptionHandler, - Optional<Integer> defaultTrafficTag) { - this.downloader = downloader; - this.fileStorage = fileStorage; - this.downloadExecutor = downloadExecutor; - this.authTokenProvider = authTokenProvider; - this.downloadMetadataStore = downloadMetadataStore; - this.exceptionHandler = exceptionHandler; - this.defaultTrafficTag = defaultTrafficTag; - } - - @Override - public ListenableFuture<Void> startDownloading( - com.google.android.libraries.mobiledatadownload.downloader.DownloadRequest - fileDownloaderRequest) { - String fileName = Strings.nullToEmpty(fileDownloaderRequest.fileUri().getLastPathSegment()); - - DownloadDestination downloadDestination; - try { - downloadDestination = buildDownloadDestination(fileDownloaderRequest.fileUri()); - } catch (DownloadException e) { - return Futures.immediateFailedFuture(e); + private static final String TAG = "Offroad2FileDownloader"; + + private final Downloader downloader; + private final SynchronousFileStorage fileStorage; + private final Executor downloadExecutor; + private final DownloadMetadataStore downloadMetadataStore; + private final ExceptionHandler exceptionHandler; + // private final Optional<Supplier<CookieJar>> cookieJarSupplierOptional; + private final Optional<Integer> defaultTrafficTag; + @Nullable + private final OAuthTokenProvider authTokenProvider; + + public Offroad2FileDownloader( + Downloader downloader, + SynchronousFileStorage fileStorage, + Executor downloadExecutor, + @Nullable OAuthTokenProvider authTokenProvider, + DownloadMetadataStore downloadMetadataStore, + ExceptionHandler exceptionHandler, +// Optional<Supplier<CookieJar>> cookieJarSupplierOptional, + Optional<Integer> defaultTrafficTag) { + this.downloader = downloader; + this.fileStorage = fileStorage; + this.downloadExecutor = downloadExecutor; + this.authTokenProvider = authTokenProvider; + this.downloadMetadataStore = downloadMetadataStore; + this.exceptionHandler = exceptionHandler; +// this.cookieJarSupplierOptional = cookieJarSupplierOptional; + this.defaultTrafficTag = defaultTrafficTag; } - DownloadRequest offroad2DownloadRequest = - buildDownloadRequest(fileDownloaderRequest, downloadDestination); - - FluentFuture<DownloadResult> resultFuture = downloader.execute(offroad2DownloadRequest); - - LogUtil.d( - "%s: Data download scheduled for file: %s", TAG, fileDownloaderRequest.urlToDownload()); - - return PropagatedFluentFuture.from(resultFuture) - .catchingAsync( - Exception.class, - cause -> { - LogUtil.d( - cause, - "%s: Failed to download file %s due to: %s", - TAG, - fileName, - Strings.nullToEmpty(cause.getMessage())); - - DownloadException exception = - exceptionHandler.mapToDownloadException("failure in download!", cause); - - return Futures.immediateFailedFuture(exception); - }, - downloadExecutor) - .transformAsync( - (DownloadResult result) -> { - LogUtil.d( - "%s: Downloaded file %s, bytes written: %d", - TAG, fileName, result.bytesWritten()); - return PropagatedFutures.catchingAsync( - downloadMetadataStore.delete(fileDownloaderRequest.fileUri()), - Exception.class, - e -> { - // Failing to clean up metadata shouldn't cause a failure in the future, log and - // return void. - LogUtil.d(e, "%s: Failed to cleanup metadata", TAG); - return Futures.immediateVoidFuture(); - }, - downloadExecutor); - }, - downloadExecutor); - } - - @Override - public ListenableFuture<CheckContentChangeResponse> isContentChanged( - CheckContentChangeRequest checkContentChangeRequest) { - return Futures.immediateFailedFuture( - new UnsupportedOperationException( - "Checking for content changes is currently unsupported for Downloader2")); - } - - private DownloadDestination buildDownloadDestination(Uri destinationUri) - throws DownloadException { - try { - // Create DownloadDestination using mobstore - return fileStorage.open( - destinationUri, DownloadDestinationOpener.create(downloadMetadataStore)); - } catch (IOException e) { - if (e instanceof MalformedUriException || e.getCause() instanceof IllegalArgumentException) { - LogUtil.e("%s: The file uri is invalid, uri = %s", TAG, destinationUri); - throw DownloadException.builder() - .setDownloadResultCode(DownloadResultCode.MALFORMED_FILE_URI_ERROR) - .setCause(e) - .build(); - } else { - LogUtil.e(e, "%s: Unable to create DownloadDestination for file %s", TAG, destinationUri); - // TODO: the result code is the most equivalent to downloader1 -- consider - // creating a separate result code that's more appropriate for downloader2. - throw DownloadException.builder() - .setDownloadResultCode( - DownloadResultCode.UNABLE_TO_CREATE_MOBSTORE_RESPONSE_WRITER_ERROR) - .setCause(e) - .build(); - } + @Override + public ListenableFuture<Void> startDownloading( + com.google.android.libraries.mobiledatadownload.downloader.DownloadRequest + fileDownloaderRequest) { + String fileName = Strings.nullToEmpty(fileDownloaderRequest.fileUri().getLastPathSegment()); + + DownloadDestination downloadDestination; + try { + downloadDestination = buildDownloadDestination(fileDownloaderRequest.fileUri()); + } catch (DownloadException e) { + return Futures.immediateFailedFuture(e); + } + + DownloadRequest offroad2DownloadRequest = + buildDownloadRequest(fileDownloaderRequest, downloadDestination); + + FluentFuture<DownloadResult> resultFuture = downloader.execute(offroad2DownloadRequest); + + LogUtil.d( + "%s: Data download scheduled for file: %s", TAG, + fileDownloaderRequest.urlToDownload()); + + return PropagatedFluentFuture.from(resultFuture) + .catchingAsync( + Exception.class, + cause -> { + LogUtil.d( + cause, + "%s: Failed to download file %s due to: %s", + TAG, + fileName, + Strings.nullToEmpty(cause.getMessage())); + + DownloadException exception = + exceptionHandler.mapToDownloadException("failure in download!", + cause); + + return Futures.immediateFailedFuture(exception); + }, + downloadExecutor) + .transformAsync( + (DownloadResult result) -> { + LogUtil.d( + "%s: Downloaded file %s, bytes written: %d", + TAG, fileName, result.bytesWritten()); + return PropagatedFutures.catchingAsync( + downloadMetadataStore.delete(fileDownloaderRequest.fileUri()), + Exception.class, + e -> { + // Failing to clean up metadata shouldn't cause a failure + // in the future, log and + // return void. + LogUtil.d(e, "%s: Failed to cleanup metadata", TAG); + return Futures.immediateVoidFuture(); + }, + downloadExecutor); + }, + downloadExecutor); } - } - - private DownloadRequest buildDownloadRequest( - com.google.android.libraries.mobiledatadownload.downloader.DownloadRequest - fileDownloaderRequest, - DownloadDestination downloadDestination) { - DownloadRequest.Builder requestBuilder = - downloader.newRequestBuilder( - URI.create(fileDownloaderRequest.urlToDownload()), downloadDestination); - - requestBuilder.setOAuthTokenProvider(authTokenProvider); - - if (com.google.android.libraries.mobiledatadownload.downloader.DownloadConstraints - .NETWORK_CONNECTED - == fileDownloaderRequest.downloadConstraints()) { - requestBuilder.setDownloadConstraints(DownloadConstraints.NETWORK_CONNECTED); - } else { - // Use all network types except cellular and require unmetered network. - requestBuilder.setDownloadConstraints( - DownloadConstraints.builder() - .addRequiredNetworkType(NetworkType.WIFI) - .addRequiredNetworkType(NetworkType.ETHERNET) - .addRequiredNetworkType(NetworkType.BLUETOOTH) - .setRequireUnmeteredNetwork(true) - .build()); + + @Override + public ListenableFuture<CheckContentChangeResponse> isContentChanged( + CheckContentChangeRequest checkContentChangeRequest) { + return Futures.immediateFailedFuture( + new UnsupportedOperationException( + "Checking for content changes is currently unsupported for Downloader2")); } - if (fileDownloaderRequest.trafficTag() > 0) { + private DownloadDestination buildDownloadDestination(Uri destinationUri) + throws DownloadException { + try { + // Create DownloadDestination using mobstore + // NOTE: the use of DirectExecutor here should be fine since all async operations + // of DownloadDestination happen within Downloader2 IOExecutor. Consider replacing + // this with + // lightweight executor. + return fileStorage.open( + destinationUri, + DownloadDestinationOpener.create(downloadMetadataStore)); + } catch (IOException e) { + if (e instanceof MalformedUriException + || e.getCause() instanceof IllegalArgumentException) { + LogUtil.e("%s: The file uri is invalid, uri = %s", TAG, destinationUri); + throw DownloadException.builder() + .setDownloadResultCode(DownloadResultCode.MALFORMED_FILE_URI_ERROR) + .setCause(e) + .build(); + } else { + LogUtil.e(e, "%s: Unable to create DownloadDestination for file %s", TAG, + destinationUri); + // TODO: the result code is the most equivalent to downloader1 -- consider + // creating a separate result code that's more appropriate for downloader2. + throw DownloadException.builder() + .setDownloadResultCode( + DownloadResultCode.UNABLE_TO_CREATE_MOBSTORE_RESPONSE_WRITER_ERROR) + .setCause(e) + .build(); + } + } + } + + private DownloadRequest buildDownloadRequest( + com.google.android.libraries.mobiledatadownload.downloader.DownloadRequest + fileDownloaderRequest, + DownloadDestination downloadDestination) { + DownloadRequest.Builder requestBuilder = + downloader.newRequestBuilder( + URI.create(fileDownloaderRequest.urlToDownload()), downloadDestination); + +// if (cookieJarSupplierOptional.isPresent()) { +// requestBuilder.setCookieJar(cookieJarSupplierOptional.get().get()); +// } + + requestBuilder.setOAuthTokenProvider(authTokenProvider); + + if (com.google.android.libraries.mobiledatadownload.downloader.DownloadConstraints + .NETWORK_CONNECTED + == fileDownloaderRequest.downloadConstraints()) { + requestBuilder.setDownloadConstraints(DownloadConstraints.NETWORK_CONNECTED); + } else { + // Use all network types except cellular and require unmetered network. + requestBuilder.setDownloadConstraints( + DownloadConstraints.builder() + .addRequiredNetworkType(NetworkType.WIFI) + .addRequiredNetworkType(NetworkType.ETHERNET) + .addRequiredNetworkType(NetworkType.BLUETOOTH) + .setRequireUnmeteredNetwork(true) + .build()); + } + + // TODO(b/237653774): Enable traffic tagging. + /*if (fileDownloaderRequest.trafficTag() > 0) { // Prefer traffic tag from request. requestBuilder.setTrafficStatsTag(fileDownloaderRequest.trafficTag()); } else if (defaultTrafficTag.isPresent() && defaultTrafficTag.get() > 0) { // Use default traffic tag as a fallback if present. requestBuilder.setTrafficStatsTag(defaultTrafficTag.get()); - } + }*/ - for (Pair<String, String> header : fileDownloaderRequest.extraHttpHeaders()) { - requestBuilder.addHeader(header.first, header.second); - } + for (Pair<String, String> header : fileDownloaderRequest.extraHttpHeaders()) { + requestBuilder.addHeader(header.first, header.second); + } - return requestBuilder.build(); - } + return requestBuilder.build(); + } } diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/ThrottlingExecutor.java b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/ThrottlingExecutor.java index 9cebf11..bbea425 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/ThrottlingExecutor.java +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/ThrottlingExecutor.java @@ -18,10 +18,10 @@ package com.google.android.libraries.mobiledatadownload.downloader.offroad; import static com.google.common.base.Preconditions.checkNotNull; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; +import com.google.errorprone.annotations.concurrent.GuardedBy; import java.util.ArrayDeque; import java.util.Queue; import java.util.concurrent.Executor; -import javax.annotation.concurrent.GuardedBy; /** * Passes tasks to a delegate {@link Executor} for execution, ensuring that no more than a fixed diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/annotations/BUILD b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/annotations/BUILD new file mode 100644 index 0000000..38f24ec --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/annotations/BUILD @@ -0,0 +1,33 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +load("@build_bazel_rules_android//android:rules.bzl", "android_library") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = [ + "//visibility:public", + ], + licenses = ["notice"], +) + +android_library( + name = "downloader2", + srcs = [ + "DownloaderFollowRedirectsImmediately.java", + ], + deps = [ + "@com_google_dagger", + "@javax_inject", + ], +) diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/annotations/DownloaderFollowRedirectsImmediately.java b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/annotations/DownloaderFollowRedirectsImmediately.java new file mode 100644 index 0000000..2f053fb --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/annotations/DownloaderFollowRedirectsImmediately.java @@ -0,0 +1,35 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.downloader.offroad.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Target; +import javax.inject.Qualifier; + +/** + * A Flag that controls whether the url engines registered to Downloader should follow redirects + * immediately. + * + * <p>In most common cases, this flag should be true, but there are some features which require this + * flag to be false (such as when providing Cookies on redirect requests is required). + * + * <p>NOTE: This flag will be calculated in MDD's {@link BaseFileDownloaderDepsModule} based on + * other client-provided dependencies, so clients do not have to provide a binding for the flag + * itself. + */ +@Target({ElementType.PARAMETER, ElementType.METHOD, ElementType.FIELD}) +@Qualifier +public @interface DownloaderFollowRedirectsImmediately {} diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/BUILD b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/BUILD index 60814e8..91b9524 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2/BUILD b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2/BUILD index 13f05e0..4e2b70f 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -34,7 +35,6 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/file/integration/downloader:downloader2", "//java/com/google/android/libraries/mobiledatadownload/monitor:DownloadProgressMonitor", - "@androidx_annotation_annotation", "@com_google_code_findbugs_jsr305", "@com_google_dagger", "@com_google_guava_guava", @@ -47,9 +47,13 @@ android_library( name = "base_deps", srcs = ["BaseFileDownloaderDepsModule.java"], deps = [ + "//java/com/google/android/libraries/mobiledatadownload/annotations", "//java/com/google/android/libraries/mobiledatadownload/downloader/offroad:ExceptionHandler", - "@androidx_annotation_annotation", + "//java/com/google/android/libraries/mobiledatadownload/downloader/offroad/annotations:downloader2", + "//java/com/google/android/libraries/mobiledatadownload/internal/util:DirectoryUtil", "@com_google_dagger", + "@com_google_guava_guava", "@downloader", + "@javax_inject", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2/BaseFileDownloaderDepsModule.java b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2/BaseFileDownloaderDepsModule.java index f98d13f..cac3bf4 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2/BaseFileDownloaderDepsModule.java +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2/BaseFileDownloaderDepsModule.java @@ -15,9 +15,9 @@ */ package com.google.android.libraries.mobiledatadownload.downloader.offroad.dagger.downloader2; -import androidx.annotation.VisibleForTesting; import com.google.android.downloader.UrlEngine; import com.google.android.libraries.mobiledatadownload.downloader.offroad.ExceptionHandler; + import dagger.BindsOptionalOf; import dagger.Module; @@ -30,24 +30,43 @@ import dagger.Module; * used across all FileDownloaders backed by Android Downloader2. */ @Module -@VisibleForTesting public abstract class BaseFileDownloaderDepsModule { - /** - * Platform specific {@link ExceptionHandler}. - * - * <p>If no specific exception handler is available, the default one will be used. - */ - @BindsOptionalOf - abstract ExceptionHandler platformSpecificExceptionHandler(); + /** + * Platform specific {@link ExceptionHandler}. + * + * <p>If no specific exception handler is available, the default one will be used. + */ + @BindsOptionalOf + abstract ExceptionHandler platformSpecificExceptionHandler(); + + /** + * Platform specific {@link UrlEngine}. + * + * <p>If no specific engine is provided, the platform engine will be used. + */ + @BindsOptionalOf + abstract UrlEngine platformSpecificUrlEngine(); - /** - * Platform specific {@link UrlEngine}. - * - * <p>If no specific engine is provided, the platform engine will be used. - */ - @BindsOptionalOf - abstract UrlEngine platformSpecificUrlEngine(); + /** + * Optional {@link CookieJar} which will be supplied to each download request. + * + * <p>If no cookie jar is provided, no cookie handling will be performed. + * + * <p>NOTE: CookieJar support is only available for Cronet at this time. // TODO(b/254955843) + * : Add + * support for platform/okhttp2/okhttp3 engines + */ +// @BindsOptionalOf +// abstract Supplier<CookieJar> requestCookieJarSupplier(); - private BaseFileDownloaderDepsModule() {} + /** Calculate whether or not we should follow redirects immediately. */ +// @Provides +// @DownloaderFollowRedirectsImmediately +// static boolean provideFollowRedirectsImmediatelyFlag( +// Optional<Supplier<CookieJar>> cookieJarSupplier) { +// return !cookieJarSupplier.isPresent(); +// } + private BaseFileDownloaderDepsModule() { + } } diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2/BaseFileDownloaderModule.java b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2/BaseFileDownloaderModule.java index 425608c..b518574 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2/BaseFileDownloaderModule.java +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2/BaseFileDownloaderModule.java @@ -18,12 +18,11 @@ package com.google.android.libraries.mobiledatadownload.downloader.offroad.dagge import static com.google.common.util.concurrent.Futures.immediateFuture; import android.content.Context; -import androidx.annotation.VisibleForTesting; + import com.google.android.downloader.AndroidConnectivityHandler; import com.google.android.downloader.Downloader; import com.google.android.downloader.Downloader.StateChangeCallback; import com.google.android.downloader.FloggerDownloaderLogger; -import com.google.android.downloader.PlatformAndroidTrafficStatsTagger; import com.google.android.downloader.PlatformUrlEngine; import com.google.android.downloader.UrlEngine; import com.google.android.libraries.mobiledatadownload.Flags; @@ -41,14 +40,17 @@ import com.google.android.libraries.mobiledatadownload.monitor.DownloadProgressM import com.google.common.base.Optional; import com.google.common.base.Supplier; import com.google.common.util.concurrent.ListeningExecutorService; + +import java.util.concurrent.ScheduledExecutorService; + +import javax.annotation.Nullable; +import javax.inject.Singleton; + import dagger.Lazy; import dagger.Module; import dagger.Provides; import dagger.multibindings.IntoMap; import dagger.multibindings.StringKey; -import java.util.concurrent.ScheduledExecutorService; -import javax.annotation.Nullable; -import javax.inject.Singleton; /** * Dagger module for providing FileDownloader that uses Android Downloader2. @@ -58,121 +60,136 @@ import javax.inject.Singleton; * module assumes is available to bind into. */ @Module( - includes = { - BaseOffroadFileDownloaderModule.class, - BaseFileDownloaderDepsModule.class, - }) -@VisibleForTesting + includes = { + BaseOffroadFileDownloaderModule.class, + BaseFileDownloaderDepsModule.class, + }) public abstract class BaseFileDownloaderModule { - @Provides - @Singleton - @IntoMap - @StringKey("https") - static Supplier<FileDownloader> provideFileDownloader( - Context context, - @MddDownloadExecutor ScheduledExecutorService downloadExecutor, - @MddControlExecutor ListeningExecutorService controlExecutor, - SynchronousFileStorage fileStorage, - DownloadMetadataStore downloadMetadataStore, - Optional<DownloadProgressMonitor> downloadProgressMonitor, - Optional<Lazy<UrlEngine>> urlEngineOptional, - Optional<Lazy<ExceptionHandler>> exceptionHandlerOptional, - Optional<Lazy<OAuthTokenProvider>> authTokenProviderOptional, - @SocketTrafficTag Optional<Integer> trafficTag, - Flags flags) { - return () -> - createOffroad2FileDownloader( - context, - downloadExecutor, - controlExecutor, - fileStorage, - downloadMetadataStore, - downloadProgressMonitor, - urlEngineOptional, - exceptionHandlerOptional, - authTokenProviderOptional, - trafficTag, - flags); - } - - @VisibleForTesting - public static Offroad2FileDownloader createOffroad2FileDownloader( - Context context, - ScheduledExecutorService downloadExecutor, - ListeningExecutorService controlExecutor, - SynchronousFileStorage fileStorage, - DownloadMetadataStore downloadMetadataStore, - Optional<DownloadProgressMonitor> downloadProgressMonitor, - Optional<Lazy<UrlEngine>> urlEngineOptional, - Optional<Lazy<ExceptionHandler>> exceptionHandlerOptional, - Optional<Lazy<OAuthTokenProvider>> authTokenProviderOptional, - Optional<Integer> trafficTag, - Flags flags) { - @Nullable - com.google.android.downloader.OAuthTokenProvider authTokenProvider = - authTokenProviderOptional.isPresent() - ? convertToDownloaderAuthTokenProvider(authTokenProviderOptional.get().get()) - : null; - - ExceptionHandler handler = - exceptionHandlerOptional.transform(Lazy::get).or(ExceptionHandler.withDefaultHandling()); - - UrlEngine urlEngine; - if (urlEngineOptional.isPresent()) { - urlEngine = urlEngineOptional.get().get(); - } else { - // Use {@link PlatformUrlEngine} if one was not provided. - urlEngine = - new PlatformUrlEngine( - controlExecutor, - /* connectTimeoutMs = */ flags.timeToWaitForDownloader(), - /* readTimeoutMs = */ flags.timeToWaitForDownloader(), - new PlatformAndroidTrafficStatsTagger()); + @Provides + @Singleton + @IntoMap + @StringKey("https") + static Supplier<FileDownloader> provideFileDownloader( + Context context, + @MddDownloadExecutor ScheduledExecutorService downloadExecutor, + @MddControlExecutor ListeningExecutorService controlExecutor, + SynchronousFileStorage fileStorage, + DownloadMetadataStore downloadMetadataStore, + Optional<DownloadProgressMonitor> downloadProgressMonitor, + Optional<Lazy<UrlEngine>> urlEngineOptional, + Optional<Lazy<ExceptionHandler>> exceptionHandlerOptional, + Optional<Lazy<OAuthTokenProvider>> authTokenProviderOptional, +// Optional<Supplier<CookieJar>> cookieJarSupplierOptional, + @SocketTrafficTag Optional<Integer> trafficTag, + Flags flags) { + return () -> + createOffroad2FileDownloader( + context, + downloadExecutor, + controlExecutor, + fileStorage, + downloadMetadataStore, + downloadProgressMonitor, + urlEngineOptional, + exceptionHandlerOptional, + authTokenProviderOptional, +// cookieJarSupplierOptional, + trafficTag, + flags); } - AndroidConnectivityHandler connectivityHandler = - new AndroidConnectivityHandler( - context, downloadExecutor, /* timeoutMillis = */ flags.timeToWaitForDownloader()); - - FloggerDownloaderLogger logger = new FloggerDownloaderLogger(); - - Downloader downloader = - new Downloader.Builder() - .withIOExecutor(controlExecutor) - .withConnectivityHandler(connectivityHandler) - .withMaxConcurrentDownloads(flags.downloaderMaxThreads()) - .withLogger(logger) - .addUrlEngine("https", urlEngine) - .build(); - - if (downloadProgressMonitor.isPresent()) { - // Wire up downloader's state changes to DownloadProgressMonitor to handle connectivity - // pauses. - StateChangeCallback callback = - state -> { - if (state.getNumDownloadsPendingConnectivity() > 0 - && state.getNumDownloadsInFlight() == 0) { - // Handle network connectivity pauses - downloadProgressMonitor.get().pausedForConnectivity(); - } - }; - downloader.registerStateChangeCallback(callback, controlExecutor); + /** + * Manual provider of Offroad2FileDownloader. + * + * <p>NOTE: This method should only be used when manually wiring up dependencies, such as when + * dagger/hilt are not available. If using dagger/hilt, this method is not needed. By + * registering + * this module in the dagger graph, the above @Provides method will automatically provide this + * dependency. + */ + public static Offroad2FileDownloader createOffroad2FileDownloader( + Context context, + ScheduledExecutorService downloadExecutor, + ListeningExecutorService controlExecutor, + SynchronousFileStorage fileStorage, + DownloadMetadataStore downloadMetadataStore, + Optional<DownloadProgressMonitor> downloadProgressMonitor, + Optional<Lazy<UrlEngine>> urlEngineOptional, + Optional<Lazy<ExceptionHandler>> exceptionHandlerOptional, + Optional<Lazy<OAuthTokenProvider>> authTokenProviderOptional, +// Optional<Supplier<CookieJar>> cookieJarSupplierOptional, + Optional<Integer> trafficTag, + Flags flags) { + @Nullable + com.google.android.downloader.OAuthTokenProvider authTokenProvider = + authTokenProviderOptional.isPresent() + ? convertToDownloaderAuthTokenProvider( + authTokenProviderOptional.get().get()) + : null; + + ExceptionHandler handler = + exceptionHandlerOptional.transform(Lazy::get).or( + ExceptionHandler.withDefaultHandling()); + + UrlEngine urlEngine; + if (urlEngineOptional.isPresent()) { + urlEngine = urlEngineOptional.get().get(); + } else { + // Use {@link PlatformUrlEngine} if one was not provided. + urlEngine = + new PlatformUrlEngine( + controlExecutor, + /* connectTimeoutMs = */ flags.timeToWaitForDownloader(), + /* readTimeoutMs = */ flags.timeToWaitForDownloader() + ); + } + + AndroidConnectivityHandler connectivityHandler = + new AndroidConnectivityHandler( + context, downloadExecutor, /* timeoutMillis= */ + flags.timeToWaitForDownloader()); + + FloggerDownloaderLogger logger = new FloggerDownloaderLogger(); + + Downloader downloader = + new Downloader.Builder() + .withIOExecutor(controlExecutor) + .withConnectivityHandler(connectivityHandler) + .withMaxConcurrentDownloads(flags.downloaderMaxThreads()) + .withLogger(logger) + .addUrlEngine("https", urlEngine) + .build(); + + if (downloadProgressMonitor.isPresent()) { + // Wire up downloader's state changes to DownloadProgressMonitor to handle connectivity + // pauses. + StateChangeCallback callback = + state -> { + if (state.getNumDownloadsPendingConnectivity() > 0 + && state.getNumDownloadsInFlight() == 0) { + // Handle network connectivity pauses + downloadProgressMonitor.get().pausedForConnectivity(); + } + }; + downloader.registerStateChangeCallback(callback, controlExecutor); + } + + return new Offroad2FileDownloader( + downloader, + fileStorage, + downloadExecutor, + authTokenProvider, + downloadMetadataStore, + handler, +// cookieJarSupplierOptional, + trafficTag); } - return new Offroad2FileDownloader( - downloader, - fileStorage, - downloadExecutor, - authTokenProvider, - downloadMetadataStore, - handler, - trafficTag); - } - - private static com.google.android.downloader.OAuthTokenProvider - convertToDownloaderAuthTokenProvider(OAuthTokenProvider authTokenProvider) { - return uri -> immediateFuture(authTokenProvider.provideOAuthToken(uri.toString())); - } - - private BaseFileDownloaderModule() {} + private static com.google.android.downloader.OAuthTokenProvider + convertToDownloaderAuthTokenProvider(OAuthTokenProvider authTokenProvider) { + return uri -> immediateFuture(authTokenProvider.provideOAuthToken(uri.toString())); + } + + private BaseFileDownloaderModule() { + } } diff --git a/java/com/google/android/libraries/mobiledatadownload/file/BUILD b/java/com/google/android/libraries/mobiledatadownload/file/BUILD index 34950b9..d3da9ef 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/file/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], diff --git a/java/com/google/android/libraries/mobiledatadownload/file/OpenContext.java b/java/com/google/android/libraries/mobiledatadownload/file/OpenContext.java index ddcb968..486544d 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/OpenContext.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/OpenContext.java @@ -20,6 +20,7 @@ import com.google.android.libraries.mobiledatadownload.file.spi.Backend; import com.google.android.libraries.mobiledatadownload.file.spi.Monitor; import com.google.android.libraries.mobiledatadownload.file.spi.Transform; import com.google.common.collect.Iterables; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -51,31 +52,37 @@ public final class OpenContext { private Builder() {} + @CanIgnoreReturnValue Builder setStorage(SynchronousFileStorage storage) { this.storage = storage; return this; } + @CanIgnoreReturnValue Builder setBackend(Backend backend) { this.backend = backend; return this; } + @CanIgnoreReturnValue Builder setTransforms(List<Transform> transforms) { this.transforms = transforms; return this; } + @CanIgnoreReturnValue Builder setMonitors(List<Monitor> monitors) { this.monitors = monitors; return this; } + @CanIgnoreReturnValue Builder setEncodedUri(Uri encodedUri) { this.encodedUri = encodedUri; return this; } + @CanIgnoreReturnValue Builder setOriginalUri(Uri originalUri) { this.originalUri = originalUri; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/backends/AndroidFileBackend.java b/java/com/google/android/libraries/mobiledatadownload/file/backends/AndroidFileBackend.java index e1d8b9d..67ebbc8 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/backends/AndroidFileBackend.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/backends/AndroidFileBackend.java @@ -28,12 +28,13 @@ import com.google.android.libraries.mobiledatadownload.file.common.MalformedUriE import com.google.android.libraries.mobiledatadownload.file.common.internal.Preconditions; import com.google.android.libraries.mobiledatadownload.file.spi.Backend; import com.google.android.libraries.mobiledatadownload.file.spi.ForwardingBackend; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import com.google.errorprone.annotations.concurrent.GuardedBy; import java.io.Closeable; import java.io.File; import java.io.IOException; import java.io.InputStream; import javax.annotation.Nullable; -import javax.annotation.concurrent.GuardedBy; /** A backend that implements "android:" scheme using {@link JavaFileBackend}. */ public final class AndroidFileBackend extends ForwardingBackend { @@ -92,6 +93,7 @@ public final class AndroidFileBackend extends ForwardingBackend { * than your own. The only methods called on {@code remoteBackend} are {@link #openForRead} and * {@link #openForNativeRead}, though this may expand in the future. Defaults to {@code null}. */ + @CanIgnoreReturnValue public Builder setRemoteBackend(Backend remoteBackend) { this.remoteBackend = remoteBackend; return this; @@ -101,6 +103,7 @@ public final class AndroidFileBackend extends ForwardingBackend { * Sets the {@link AccountManager} invoked to resolve "managed" URIs. Defaults to {@code null}, * in which case operations on "managed" URIs will fail. */ + @CanIgnoreReturnValue public Builder setAccountManager(AccountManager accountManager) { this.accountManager = accountManager; return this; @@ -111,6 +114,7 @@ public final class AndroidFileBackend extends ForwardingBackend { * injection is only necessary if there are multiple backend instances in the same process and * there's a risk of them acquiring a lock on the same underlying file. */ + @CanIgnoreReturnValue public Builder setLockScope(LockScope lockScope) { Preconditions.checkArgument( backend == null, diff --git a/java/com/google/android/libraries/mobiledatadownload/file/backends/AndroidUri.java b/java/com/google/android/libraries/mobiledatadownload/file/backends/AndroidUri.java index da6bc2e..26b0107 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/backends/AndroidUri.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/backends/AndroidUri.java @@ -24,6 +24,7 @@ import com.google.android.libraries.mobiledatadownload.file.common.internal.Lite import com.google.android.libraries.mobiledatadownload.file.common.internal.Preconditions; import com.google.android.libraries.mobiledatadownload.file.transforms.TransformProtos; import com.google.common.collect.ImmutableList; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.mobiledatadownload.TransformProto; import java.io.File; import java.util.Arrays; @@ -149,42 +150,51 @@ public final class AndroidUri { /** * Sets the package to use in the android uri AUTHORITY. Default is context.getPackageName(). */ + @CanIgnoreReturnValue public Builder setPackage(String packageName) { this.packageName = packageName; return this; } + @CanIgnoreReturnValue private Builder setLocation(String location) { AndroidUri.validateLocation(location); this.location = location; return this; } + @CanIgnoreReturnValue public Builder setManagedLocation() { return setLocation(MANAGED_LOCATION); } + @CanIgnoreReturnValue public Builder setExternalLocation() { return setLocation(EXTERNAL_LOCATION); } + @CanIgnoreReturnValue public Builder setDirectBootFilesLocation() { return setLocation(DIRECT_BOOT_FILES_LOCATION); } + @CanIgnoreReturnValue public Builder setDirectBootCacheLocation() { return setLocation(DIRECT_BOOT_CACHE_LOCATION); } /** Internal location, aka "files", is the default location. */ + @CanIgnoreReturnValue public Builder setInternalLocation() { return setLocation(FILES_LOCATION); } + @CanIgnoreReturnValue public Builder setCacheLocation() { return setLocation(CACHE_LOCATION); } + @CanIgnoreReturnValue public Builder setModule(String module) { AndroidUri.validateModule(module); this.module = module; @@ -210,6 +220,7 @@ public final class AndroidUri { * @param account The account to set. * @return The fluent Builder. */ + @CanIgnoreReturnValue public Builder setAccount(Account account) { AccountSerialization.serialize(account); // performs validation internally this.account = account; @@ -220,6 +231,7 @@ public final class AndroidUri { * Sets the component of the path after location, module and account. A single leading slash * will be trimmed if present. */ + @CanIgnoreReturnValue public Builder setRelativePath(String relativePath) { if (relativePath.startsWith("/")) { relativePath = relativePath.substring(1); @@ -233,6 +245,7 @@ public final class AndroidUri { * Updates builder with multiple fields from file param: location, module, account and relative * path. This method will fail on "managed" paths (see {@link fromFile(File, AccountManager)}). */ + @CanIgnoreReturnValue public Builder fromFile(File file) { return fromAbsolutePath(file.getAbsolutePath(), /* accountManager= */ null); } @@ -241,6 +254,7 @@ public final class AndroidUri { * Updates builder with multiple fields from file param: location, module, account and relative * path. A non-null {@code accountManager} is required to handle "managed" paths. */ + @CanIgnoreReturnValue public Builder fromFile(File file, @Nullable AccountManager accountManager) { return fromAbsolutePath(file.getAbsolutePath(), accountManager); } @@ -250,6 +264,7 @@ public final class AndroidUri { * relative path. This method will fail on "managed" paths (see {@link fromAbsolutePath(String, * AccountManager)}). */ + @CanIgnoreReturnValue public Builder fromAbsolutePath(String absolutePath) { return fromAbsolutePath(absolutePath, /* accountManager= */ null); } @@ -259,6 +274,7 @@ public final class AndroidUri { * relative path. A non-null {@code accountManager} is required to handle "managed" paths. */ // TODO(b/129467051): remove requirement for segments after 0th (logical location) + @CanIgnoreReturnValue public Builder fromAbsolutePath(String absolutePath, @Nullable AccountManager accountManager) { // Get the file's path within internal files, /module/account</relativePath> File filesDir = AndroidFileEnvironment.getFilesDirWithPreNWorkaround(context); @@ -341,6 +357,7 @@ public final class AndroidUri { return this; } + @CanIgnoreReturnValue public Builder withTransform(TransformProto.Transform spec) { encodedSpecs.add(TransformProtos.toEncodedSpec(spec)); return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/backends/AndroidUriAdapter.java b/java/com/google/android/libraries/mobiledatadownload/file/backends/AndroidUriAdapter.java index 7f42232..c7c8ff1 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/backends/AndroidUriAdapter.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/backends/AndroidUriAdapter.java @@ -28,7 +28,7 @@ import javax.annotation.Nullable; /** * Adapter for converting "android:" URIs into java.io.File. This is considered dangerous since it - * ignores parts of the Uri at the caller's peril, and thus is only available to allowlisted clients + * ignores parts of the Uri at the caller's peril, and thus is only available to whitelisted clients * (mostly internal). */ public final class AndroidUriAdapter implements UriAdapter { diff --git a/java/com/google/android/libraries/mobiledatadownload/file/backends/BUILD b/java/com/google/android/libraries/mobiledatadownload/file/backends/BUILD index 0e919e5..2abed92 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/backends/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/file/backends/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -52,6 +53,7 @@ android_library( "//proto:transform_java_proto_lite", "@androidx_annotation_annotation", "@com_google_code_findbugs_jsr305", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], ) @@ -63,6 +65,7 @@ android_library( ], deps = [ "//java/com/google/android/libraries/mobiledatadownload/file/common", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], ) @@ -78,6 +81,7 @@ android_library( ":file_descriptor", "//java/com/google/android/libraries/mobiledatadownload/file/common", "//java/com/google/android/libraries/mobiledatadownload/file/spi", + "@com_google_code_findbugs_jsr305", "@com_google_guava_guava", ], ) @@ -92,6 +96,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file/common", "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:preconditions", "//java/com/google/android/libraries/mobiledatadownload/file/spi", + "@com_google_errorprone_error_prone_annotations", ], ) @@ -121,6 +126,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file/spi", "//java/com/google/android/libraries/mobiledatadownload/file/transforms:proto", "//proto:transform_java_proto_lite", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], ) @@ -137,6 +143,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file/spi", "//java/com/google/android/libraries/mobiledatadownload/file/transforms:proto", "//proto:transform_java_proto_lite", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], ) @@ -171,6 +178,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file/transforms:proto", "//proto:transform_java_proto_lite", "@com_google_code_findbugs_jsr305", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], ) @@ -211,6 +219,7 @@ android_library( visibility = ["//:__subpackages__"], deps = [ "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:preconditions", + "@com_google_errorprone_error_prone_annotations", # NOTE: dependency of gmscore client lib <internal> ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/file/backends/BlobStoreBackend.java b/java/com/google/android/libraries/mobiledatadownload/file/backends/BlobStoreBackend.java index 497efc0..80ae24b 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/backends/BlobStoreBackend.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/backends/BlobStoreBackend.java @@ -34,6 +34,7 @@ import java.io.OutputStream; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import javax.annotation.Nullable; /** * Backend for accessing the Android blob Sharing Service. @@ -118,7 +119,7 @@ public final class BlobStoreBackend implements Backend { * @throws IOException when there is an I/O error while writing the blob/lease. */ @Override - public OutputStream openForWrite(Uri uri) throws IOException { + public @Nullable OutputStream openForWrite(Uri uri) throws IOException { BlobUri.validateUri(uri); byte[] checksum = BlobUri.getChecksum(uri.getPath()); try { diff --git a/java/com/google/android/libraries/mobiledatadownload/file/backends/BlobUri.java b/java/com/google/android/libraries/mobiledatadownload/file/backends/BlobUri.java index 28b14e5..29fc1f1 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/backends/BlobUri.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/backends/BlobUri.java @@ -21,6 +21,7 @@ import android.text.TextUtils; import com.google.android.libraries.mobiledatadownload.file.common.MalformedUriException; import com.google.common.base.Splitter; import com.google.common.io.BaseEncoding; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.util.List; /** Helper class for "blobstore" URIs. */ @@ -151,17 +152,20 @@ public final class BlobUri { this.packageName = context.getPackageName(); } + @CanIgnoreReturnValue public Builder setBlobParameters(String checksum) { path = checksum; return this; } + @CanIgnoreReturnValue public Builder setLeaseParameters(String checksum, long expiryDateSecs) { path = checksum + LEASE_URI_SUFFIX; this.expiryDateSecs = expiryDateSecs; return this; } + @CanIgnoreReturnValue public Builder setAllLeasesParameters() { path = ALL_LEASES_PATH; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/backends/ContentResolverBackend.java b/java/com/google/android/libraries/mobiledatadownload/file/backends/ContentResolverBackend.java index 5c31747..d5e8da9 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/backends/ContentResolverBackend.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/backends/ContentResolverBackend.java @@ -22,6 +22,7 @@ import android.util.Pair; import com.google.android.libraries.mobiledatadownload.file.common.MalformedUriException; import com.google.android.libraries.mobiledatadownload.file.common.internal.Preconditions; import com.google.android.libraries.mobiledatadownload.file.spi.Backend; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.Closeable; import java.io.IOException; import java.io.InputStream; @@ -39,7 +40,7 @@ import java.io.InputStream; * * <p>NOTE: In most cases, you'll want to use the GmsClientBackend for accessing files from GMS * core. This backend is used to access files from other Apps. Since there are possible security - * concerns with doing so, ContentResolverBackend is restricted to the "content_resolver_allowlist". + * concerns with doing so, ContentResolverBackend is restricted to the "content_resolver_whitelist". * See <internal> for more information. */ public final class ContentResolverBackend implements Backend { @@ -67,6 +68,7 @@ public final class ContentResolverBackend implements Backend { * Tells whether this backend is expected to be embedded in another backend. If so, rewrites the * scheme to "content"; if not, requires that the scheme be "content". */ + @CanIgnoreReturnValue public Builder setEmbedded(boolean isEmbedded) { this.isEmbedded = isEmbedded; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/backends/FileUri.java b/java/com/google/android/libraries/mobiledatadownload/file/backends/FileUri.java index 58f9508..5a00ec9 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/backends/FileUri.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/backends/FileUri.java @@ -19,6 +19,7 @@ import android.net.Uri; import com.google.android.libraries.mobiledatadownload.file.common.internal.LiteTransformFragments; import com.google.android.libraries.mobiledatadownload.file.transforms.TransformProtos; import com.google.common.collect.ImmutableList; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.mobiledatadownload.TransformProto; import java.io.File; @@ -46,21 +47,25 @@ public final class FileUri { private Builder() {} + @CanIgnoreReturnValue public Builder setPath(String path) { uri.path(path); return this; } + @CanIgnoreReturnValue public Builder fromFile(File file) { uri.path(file.getAbsolutePath()); return this; } + @CanIgnoreReturnValue public Builder appendPath(String segment) { uri.appendPath(segment); return this; } + @CanIgnoreReturnValue public Builder withTransform(TransformProto.Transform spec) { encodedSpecs.add(TransformProtos.toEncodedSpec(spec)); return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/backends/FileUriAdapter.java b/java/com/google/android/libraries/mobiledatadownload/file/backends/FileUriAdapter.java index ea73f06..625e1c1 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/backends/FileUriAdapter.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/backends/FileUriAdapter.java @@ -22,7 +22,7 @@ import java.io.File; /** * Adapter for converting "file:" URIs into java.io.File. This is considered dangerous since it - * ignores parts of the Uri at the caller's peril, and thus is only available to allowlisted clients + * ignores parts of the Uri at the caller's peril, and thus is only available to whitelisted clients * (mostly internal). */ public class FileUriAdapter implements UriAdapter { diff --git a/java/com/google/android/libraries/mobiledatadownload/file/backends/GenericUriAdapter.java b/java/com/google/android/libraries/mobiledatadownload/file/backends/GenericUriAdapter.java index 5e244f2..c9d85c6 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/backends/GenericUriAdapter.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/backends/GenericUriAdapter.java @@ -22,7 +22,7 @@ import java.io.File; /** * Adapter for converting "android:" URIs into java.io.File. This is considered dangerous since it - * ignores parts of the Uri at the caller's peril, and thus is only available to allowlisted clients + * ignores parts of the Uri at the caller's peril, and thus is only available to whitelisted clients * (mostly internal). */ public final class GenericUriAdapter implements UriAdapter { diff --git a/java/com/google/android/libraries/mobiledatadownload/file/backends/MemoryUri.java b/java/com/google/android/libraries/mobiledatadownload/file/backends/MemoryUri.java index 2d69d72..7f96b32 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/backends/MemoryUri.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/backends/MemoryUri.java @@ -21,6 +21,7 @@ import com.google.android.libraries.mobiledatadownload.file.common.MalformedUriE import com.google.android.libraries.mobiledatadownload.file.common.internal.LiteTransformFragments; import com.google.android.libraries.mobiledatadownload.file.transforms.TransformProtos; import com.google.common.collect.ImmutableList; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.mobiledatadownload.TransformProto; /** @@ -45,6 +46,7 @@ public final class MemoryUri { private Builder() {} /** Sets the non-empty key to be used as a file identifier. */ + @CanIgnoreReturnValue public Builder setKey(String key) { this.key = key; return this; @@ -53,6 +55,7 @@ public final class MemoryUri { /** * Appends a transform to the Uri. Calling twice with the same transform replaces the original. */ + @CanIgnoreReturnValue public Builder withTransform(TransformProto.Transform spec) { encodedSpecs.add(TransformProtos.toEncodedSpec(spec)); return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/backends/UriAdapter.java b/java/com/google/android/libraries/mobiledatadownload/file/backends/UriAdapter.java index e9a73aa..029893b 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/backends/UriAdapter.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/backends/UriAdapter.java @@ -22,7 +22,7 @@ import java.io.File; /** * Interface for converting certain URI schemes to raw java.io.Files. Implementations of this are * considered dangerous since they ignore parts of the URI incluging the fragment at the caller's - * peril, and thus is only available to allowlisted clients (mostly internal). + * peril, and thus is only available to whitelisted clients (mostly internal). */ interface UriAdapter { /** diff --git a/java/com/google/android/libraries/mobiledatadownload/file/behaviors/BUILD b/java/com/google/android/libraries/mobiledatadownload/file/behaviors/BUILD index 5d2195a..977f913 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/behaviors/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/file/behaviors/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], diff --git a/java/com/google/android/libraries/mobiledatadownload/file/common/BUILD b/java/com/google/android/libraries/mobiledatadownload/file/common/BUILD index c183243..7b1fb08 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/common/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/file/common/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -39,6 +40,9 @@ android_library( "UnsupportedFileStorageOperation.java", ], deps = [ + "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:exponential_backoff_iterator", + "@com_google_guava_guava", + "@com_google_errorprone_error_prone_annotations", "@com_google_code_findbugs_jsr305", # NOTE: dependency of gmscore client lib <internal> ], @@ -53,6 +57,7 @@ android_library( deps = [ "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:charsets", "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:preconditions", + "@com_google_errorprone_error_prone_annotations", "@com_google_code_findbugs_jsr305", # NOTE: dependency of gmscore client lib <internal> ], diff --git a/java/com/google/android/libraries/mobiledatadownload/file/common/Fragment.java b/java/com/google/android/libraries/mobiledatadownload/file/common/Fragment.java index 5d02885..62e78e3 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/common/Fragment.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/common/Fragment.java @@ -20,6 +20,7 @@ import static com.google.android.libraries.mobiledatadownload.file.common.intern import android.net.Uri; import android.text.TextUtils; import com.google.android.libraries.mobiledatadownload.file.common.internal.Preconditions; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; import java.net.URLEncoder; @@ -128,12 +129,14 @@ public final class Fragment { } /** Adds a param. If a param with same key already exists, this replaces it. */ + @CanIgnoreReturnValue public Builder addParam(Param param) { addParam(param.toBuilder()); return this; } /** Adds a param. If a param with the same key already exist, this replaces it. */ + @CanIgnoreReturnValue public Builder addParam(Param.Builder param) { for (int i = 0; i < params.size(); i++) { if (params.get(i).key.equals(param.key)) { @@ -146,6 +149,7 @@ public final class Fragment { } /** Adds a simple param with no value. */ + @CanIgnoreReturnValue public Builder addParam(String key) { return addParam(Param.builder(key)); } @@ -266,6 +270,7 @@ public final class Fragment { * Adds a value to this param. If a value already exists with the same name, this will replace * it. */ + @CanIgnoreReturnValue public Builder addValue(ParamValue value) { addValue(value.toBuilder()); return this; @@ -275,6 +280,7 @@ public final class Fragment { * Adds a value to this param. If a value already exists with the same name, this will replace * it. */ + @CanIgnoreReturnValue public Builder addValue(ParamValue.Builder value) { for (int i = 0; i < values.size(); i++) { if (values.get(i).name.equals(value.name)) { @@ -287,6 +293,7 @@ public final class Fragment { } /** Adds a value that has no subparams. Also replaces existing value if present. */ + @CanIgnoreReturnValue public Builder addValue(String name) { return addValue(new ParamValue.Builder(name, null)); } @@ -434,6 +441,7 @@ public final class Fragment { * @param subparam * @return The subparam or null if not found. */ + @CanIgnoreReturnValue public Builder addSubParam(SubParam subparam) { for (int i = 0; i < subparams.size(); i++) { if (subparams.get(i).key.equals(subparam.key)) { @@ -452,6 +460,7 @@ public final class Fragment { * @param key The subparam key. * @param value The subparam value. */ + @CanIgnoreReturnValue public Builder addSubParam(String key, String value) { return addSubParam(new SubParam(key, value)); } diff --git a/java/com/google/android/libraries/mobiledatadownload/file/common/LockScope.java b/java/com/google/android/libraries/mobiledatadownload/file/common/LockScope.java index 00e68e0..2c68111 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/common/LockScope.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/common/LockScope.java @@ -16,11 +16,15 @@ package com.google.android.libraries.mobiledatadownload.file.common; import android.net.Uri; +import android.os.SystemClock; +import com.google.android.libraries.mobiledatadownload.file.common.internal.ExponentialBackoffIterator; +import com.google.common.base.Optional; import java.io.Closeable; import java.io.IOException; import java.io.InterruptedIOException; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; +import java.util.Iterator; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Semaphore; @@ -44,6 +48,17 @@ import javax.annotation.Nullable; */ public final class LockScope { + // NOTE(b/254717998): due to the design of Linux file lock, it would throw an IOException with + // "Resource deadlock would occur" as false alarms in some use cases. As the fix, in the case of + // such failures where error message matches with {@link DEADLOCK_ERROR_MESSAGE}, we first do + // exponential backoff to retry to get file lock, and then retry every second until it succeeds. + private static final String DEADLOCK_ERROR_MESSAGE = "Resource deadlock would occur"; + + // Wait for 10 ms if need to retry file locking for the first time + private static final Long INITIAL_WAIT_MILLIS = 10L; + // Wait for 1 minute if need to retry file locking with the upper bound wait time + private static final Long UPPER_BOUND_WAIT_MILLIS = 60_000L; + @Nullable private final ConcurrentMap<String, Semaphore> lockMap; /** @@ -109,8 +124,29 @@ public final class LockScope { /** Acquires a cross-process lock on {@code channel}. This blocks until the lock is obtained. */ public Lock fileLock(FileChannel channel, boolean shared) throws IOException { - FileLock lock = channel.lock(0L /* position */, Long.MAX_VALUE /* size */, shared); - return new FileLockImpl(lock); + Optional<FileLockImpl> fileLock = fileLockAndThrowIfNotDeadlock(channel, shared); + if (fileLock.isPresent()) { + return fileLock.get(); + } + + // if an IOException with "Resource deadlock would occur" is thrown from getting file lock, we + // will keep retrying until it succeeds + Iterator<Long> retryIterator = + ExponentialBackoffIterator.create(INITIAL_WAIT_MILLIS, UPPER_BOUND_WAIT_MILLIS); + // TODO(b/254717998): error after a number of retry attempts if needed. And possibly detect real + // deadlocks in client use cases. + while (retryIterator.hasNext()) { + long waitTime = retryIterator.next(); + SystemClock.sleep(waitTime); + + Optional<FileLockImpl> fileLockImpl = fileLockAndThrowIfNotDeadlock(channel, shared); + if (fileLockImpl.isPresent()) { + return fileLockImpl.get(); + } + } + // should never reach here because ExponentialBackoffIterator guarantees it will always hasNext, + // make builder happy + throw new IllegalStateException("should have gotten file lock and returned"); } /** @@ -136,38 +172,36 @@ public final class LockScope { return lockMap != null; } + /** + * Returns the file lock got from given channel. If gets an IOException with {@link + * DEADLOCK_ERROR_MESSAGE}, returns empty; otherwise throws the error. + */ + private static Optional<FileLockImpl> fileLockAndThrowIfNotDeadlock( + FileChannel channel, boolean shared) throws IOException { + try { + FileLock lock = channel.lock(0L /* position */, Long.MAX_VALUE /* size */, shared); + return Optional.of(new FileLockImpl(lock)); + } catch (IOException ex) { + if (!ex.getMessage().contains(DEADLOCK_ERROR_MESSAGE)) { + throw ex; + } + return Optional.absent(); + } + } + private static class FileLockImpl implements Lock { private FileLock fileLock; - private Semaphore semaphore; public FileLockImpl(FileLock fileLock) { this.fileLock = fileLock; - this.semaphore = null; - } - - /** - * @deprecated Prefer the single-argument {@code FileLockImpl(FileLock)}. - */ - @Deprecated - public FileLockImpl(FileLock fileLock, Semaphore semaphore) { - this.fileLock = fileLock; - this.semaphore = semaphore; } @Override public void release() throws IOException { - // The semaphore guards access to the fileLock, so fileLock *must* be released first. - try { - if (fileLock != null) { - fileLock.release(); - fileLock = null; - } - } finally { - if (semaphore != null) { - semaphore.release(); - semaphore = null; - } + if (fileLock != null) { + fileLock.release(); + fileLock = null; } } diff --git a/java/com/google/android/libraries/mobiledatadownload/file/common/internal/BUILD b/java/com/google/android/libraries/mobiledatadownload/file/common/internal/BUILD index 0ffd400..adc1b24 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/common/internal/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/file/common/internal/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -77,3 +78,9 @@ android_library( "@com_google_guava_guava", ], ) + +android_library( + name = "exponential_backoff_iterator", + srcs = ["ExponentialBackoffIterator.java"], + deps = ["@com_google_guava_guava"], +) diff --git a/java/com/google/android/libraries/mobiledatadownload/file/common/internal/ExponentialBackoffIterator.java b/java/com/google/android/libraries/mobiledatadownload/file/common/internal/ExponentialBackoffIterator.java new file mode 100644 index 0000000..574ff22 --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/file/common/internal/ExponentialBackoffIterator.java @@ -0,0 +1,67 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.file.common.internal; + +import static com.google.common.base.Preconditions.checkArgument; + +import java.util.Iterator; + +/** + * Provide an iterator for a infinite sequence of exponential backoffs. The sequence begins with the + * provided initial backoff and doubles up everytime a new backoff is acceessed, after the backoff + * reaches the upper bound, it always returns the upper bound backoff. + */ +public final class ExponentialBackoffIterator implements Iterator<Long> { + + /** + * Create a new instance with positive integers. {@code upperBoundBackoff} should be no less than + * {@code initialBackoff}. + */ + public static ExponentialBackoffIterator create(long initialBackoff, long upperBoundBackoff) { + checkArgument(initialBackoff > 0); + checkArgument(upperBoundBackoff >= initialBackoff); + return new ExponentialBackoffIterator(initialBackoff, upperBoundBackoff); + } + + private long nextBackoff; + private final long upperBoundBackoff; + + private ExponentialBackoffIterator(long initialBackoff, long upperBoundBackoff) { + this.nextBackoff = initialBackoff; + this.upperBoundBackoff = upperBoundBackoff; + } + + /** + * Returns if the iterator has the next delay. It always returns true because the sequence is + * infinite. + */ + @Override + public boolean hasNext() { + return true; + } + + /** Returns the next delay. */ + @Override + public Long next() { + long currentBackoff = nextBackoff; + if (nextBackoff >= upperBoundBackoff / 2) { + nextBackoff = upperBoundBackoff; + } else { + nextBackoff *= 2; + } + return currentBackoff; + } +} diff --git a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/BUILD b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/BUILD index 410729f..9085543 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/BUILD @@ -15,6 +15,7 @@ load("//tools/build_defs/testing:bzl_library.bzl", "bzl_library") load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_testonly = 1, default_visibility = ["//:__subpackages__"], licenses = ["notice"], @@ -32,6 +33,7 @@ java_library( ], deps = [ "@android_sdk_linux", + "@com_google_errorprone_error_prone_annotations", "@robolectric", ], ) @@ -78,6 +80,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:charsets", "//java/com/google/android/libraries/mobiledatadownload/file/openers:stream", "//java/com/google/android/libraries/mobiledatadownload/file/spi", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", "@junit", "@truth", @@ -102,6 +105,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file/common:fragment", "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:forwarding_stream", "//java/com/google/android/libraries/mobiledatadownload/file/spi", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", "@junit", "@truth", @@ -120,18 +124,20 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:forwarding_stream", "//java/com/google/android/libraries/mobiledatadownload/file/spi", "@com_google_code_findbugs_jsr305", + "@com_google_errorprone_error_prone_annotations", "@org_checkerframework_qual", ], ) java_lite_proto_library( name = "test_message_java_proto_lite", - deps = [":test_message_proto"], + deps = ["//java/com/google/android/libraries/mobiledatadownload/file/common/testing:test_message_proto"], ) proto_library( name = "test_message_proto", srcs = ["test_message.proto"], + deps = ["@com_google_protobuf//:timestamp_proto"], ) bzl_library( diff --git a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/BackendTestBase.java b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/BackendTestBase.java index ade1277..5e2af9c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/BackendTestBase.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/BackendTestBase.java @@ -266,7 +266,7 @@ public abstract class BackendTestBase { try (OutputStream stream = backend().openForAppend(uri)) { assertThat(stream).isInstanceOf(FileConvertible.class); File file = ((FileConvertible) stream).toFile(); - writeFileToSink(new FileOutputStream(file, /* append = */ true), TEST_CONTENT); + writeFileToSink(new FileOutputStream(file, /* append= */ true), TEST_CONTENT); } assertThat(readFileInBytes(storage(), uri)) .isEqualTo(Bytes.concat(OTHER_CONTENT, TEST_CONTENT)); diff --git a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/ExceptionTesting.java b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/ExceptionTesting.java index 77557bb..e09768d 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/ExceptionTesting.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/ExceptionTesting.java @@ -23,6 +23,7 @@ import java.util.concurrent.Future; /** Common helper utilities for testing exceptions. */ public final class ExceptionTesting { + public static <T extends Throwable> T assertThrowsAsync( Class<T> throwableType, Future<?> future) { ExecutionException executionException = assertThrows(ExecutionException.class, future::get); diff --git a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/FakeFileBackend.java b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/FakeFileBackend.java index 2581c9a..83b883d 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/FakeFileBackend.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/FakeFileBackend.java @@ -22,6 +22,7 @@ import com.google.android.libraries.mobiledatadownload.file.common.GcParam; import com.google.android.libraries.mobiledatadownload.file.common.LockScope; import com.google.android.libraries.mobiledatadownload.file.common.internal.ForwardingOutputStream; import com.google.android.libraries.mobiledatadownload.file.spi.Backend; +import com.google.errorprone.annotations.concurrent.GuardedBy; import java.io.Closeable; import java.io.File; import java.io.IOException; @@ -30,7 +31,6 @@ import java.io.OutputStream; import java.util.EnumMap; import java.util.Map; import java.util.concurrent.CountDownLatch; -import javax.annotation.concurrent.GuardedBy; import org.checkerframework.checker.nullness.qual.Nullable; /** A Fake Backend for testing. It allows overriding certain behavior. */ @@ -53,6 +53,7 @@ public class FakeFileBackend implements Backend { QUERY, // exists, isDirectory, fileSize, children, getGcParam, toFile MANAGE, // delete, rename, createDirectory, setGcParam WRITE_STREAM, // openForWrite/Append return streams that fail + EXISTS, // exists } /** @@ -212,6 +213,7 @@ public class FakeFileBackend implements Backend { @Override public boolean exists(Uri uri) throws IOException { + throwOrSuspendIf(OperationType.EXISTS); throwOrSuspendIf(OperationType.QUERY); return delegate.exists(uri); } diff --git a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/FileDescriptorLeakChecker.java b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/FileDescriptorLeakChecker.java index 981de70..5bb36b8 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/FileDescriptorLeakChecker.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/FileDescriptorLeakChecker.java @@ -25,6 +25,7 @@ import android.util.Log; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -56,11 +57,13 @@ public class FileDescriptorLeakChecker implements MethodRule { * * @param processesToMonitor The names of the processes to monitor. */ + @CanIgnoreReturnValue public FileDescriptorLeakChecker withProcessesToMonitor(List<String> processesToMonitor) { this.processesToMonitor = processesToMonitor; return this; } + @CanIgnoreReturnValue public FileDescriptorLeakChecker withFilesToMonitor(List<String> filesToMonitor) { this.filesToMonitor = filesToMonitor; return this; @@ -72,6 +75,7 @@ public class FileDescriptorLeakChecker implements MethodRule { * * @param msToWait Milliseconds the FileDescriptorLeakChecker needs to wait before retrying. */ + @CanIgnoreReturnValue public FileDescriptorLeakChecker withWaitIfFails(long msToWait) { this.msToWait = msToWait; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/build_defs.bzl b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/build_defs.bzl index eef907a..a628f20 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/build_defs.bzl +++ b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/build_defs.bzl @@ -36,9 +36,11 @@ _DEFAULT_TARGET_APIS = [ _GENERIC_DEVICE_FMT = "generic_phone:google_%s_x86_gms_stable" _EMULATOR_DIRECTORY = "//tools/android/emulated_devices/%s" +# TODO: Consider adding the local test for additional target devices def android_test_multi_api( name, target_apis = _DEFAULT_TARGET_APIS, + additional_targets = {}, **kwargs): """Simple definition for running an android_application_test against multiple API levels. @@ -56,6 +58,8 @@ def android_test_multi_api( name: Name of the "default" test target and used to derive subtargets target_apis: List of Android API levels as strings for which a test should be generated. If unspecified, 16-28 excluding 20 are used. + additional_targets: Map of additional target devices other than automatically generated ones, + with keys as target device names and values as emulator directory. **kwargs: Parameters that are passed to the generated android_application_test rule. """ @@ -67,6 +71,7 @@ def android_test_multi_api( android_test_multi_device( name = name, target_devices = target_devices, + additional_targets_map = additional_targets, **kwargs ) @@ -84,6 +89,7 @@ def android_test_multi_api( def android_test_multi_device( name, target_devices, + additional_targets_map, **kwargs): """Simple definition for running an android_application_test against multiple devices. @@ -91,14 +97,35 @@ def android_test_multi_device( name: Name of the test rule; we generate several sub-targets based on API. target_devices: List of emulators as strings for which a test should be generated. + additional_targets_map: Map of additional target devices other than automatically generated + ones, with keys as target device names and values as emulator directory. **kwargs: Parameters that are passed to the generated android_application_test rule. """ for target_device in target_devices: - sanitized_device = target_device.replace(":", "_") # ":" is invalid - test_name = "%s_%s" % (name, sanitized_device) - test_device = _EMULATOR_DIRECTORY % (target_device) - android_application_test( - name = test_name, - target_devices = [test_device], - **kwargs - ) + android_test_single_device(name, target_device, _EMULATOR_DIRECTORY, **kwargs) + for additional_target, emulator_dir in additional_targets_map.items(): + if not emulator_dir.endswith("%s"): + emulator_dir += "%s" + android_test_single_device(name, additional_target, emulator_dir, **kwargs) + +def android_test_single_device( + name, + target_device, + emulator_directory, + **kwargs): + """Simple definition for running an android_application_test against single device. + + Args: + name: Name of the test rule; we generate several sub-targets based on API. + target_device: An emulator as a string for which a test should be generated. + emulator_directory: A string representing the diretory where the emulator locates at. + **kwargs: Parameters that are passed to the generated android_application_test rule. + """ + sanitized_device = target_device.replace(":", "_") # ":" is invalid + test_name = "%s_%s" % (name, sanitized_device) + test_device = emulator_directory % (target_device) + android_application_test( + name = test_name, + target_devices = [test_device], + **kwargs + ) diff --git a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/test_message.proto b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/test_message.proto index 4611f9c..8eeed3e 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/test_message.proto +++ b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/test_message.proto @@ -16,6 +16,8 @@ syntax = "proto2"; package google.android.storage.common; +import "google/protobuf/timestamp.proto"; + option java_package = "com.google.mobiledatadownload.testing"; option java_outer_classname = "TestMessageProto"; @@ -24,6 +26,7 @@ message FooProto { optional bool boolean = 2; optional int32 integer = 3; optional bytes bytes = 4; + optional google.protobuf.Timestamp timestamp = 5; } message BarProto { diff --git a/java/com/google/android/libraries/mobiledatadownload/file/integration/downloader/BUILD b/java/com/google/android/libraries/mobiledatadownload/file/integration/downloader/BUILD index 42e34fa..e2d867d 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/integration/downloader/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/file/integration/downloader/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -30,6 +31,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/file/common", "//java/com/google/android/libraries/mobiledatadownload/file/openers:random_access_file", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "@com_google_guava_guava", "@downloader", ], diff --git a/java/com/google/android/libraries/mobiledatadownload/file/integration/downloader/DownloadDestinationOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/integration/downloader/DownloadDestinationOpener.java index 855ec85..1a3322a 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/integration/downloader/DownloadDestinationOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/integration/downloader/DownloadDestinationOpener.java @@ -72,7 +72,7 @@ public final class DownloadDestinationOpener implements Opener<DownloadDestinati private final SynchronousFileStorage fileStorage; private DownloadDestinationImpl( - Uri onDeviceUri, SynchronousFileStorage fileStorage, DownloadMetadataStore metadataStore) { + Uri onDeviceUri, SynchronousFileStorage fileStorage, DownloadMetadataStore metadataStore) { this.onDeviceUri = onDeviceUri; this.metadataStore = metadataStore; this.fileStorage = fileStorage; @@ -87,7 +87,7 @@ public final class DownloadDestinationOpener implements Opener<DownloadDestinati public DownloadMetadata readMetadata() throws IOException { synchronized (lock) { Optional<DownloadMetadata> existingMetadata = - blockingGet(metadataStore.read(onDeviceUri), "Failed to read metadata."); + blockingGet(metadataStore.read(onDeviceUri), "Failed to read metadata."); // Return existing metadata, or a new instance. return existingMetadata.or(DownloadMetadata::create); @@ -96,16 +96,16 @@ public final class DownloadDestinationOpener implements Opener<DownloadDestinati @Override public WritableByteChannel openByteChannel(long byteOffset, DownloadMetadata metadata) - throws IOException { + throws IOException { // Ensure that metadata is not null checkArgument(metadata != null, "Received null metadata to store"); // Check that offset is in range long fileSize = numExistingBytes(); checkArgument( - byteOffset >= 0 && byteOffset <= fileSize, - "Offset for write (%s) out of range of existing file size (%s bytes)", - byteOffset, - fileSize); + byteOffset >= 0 && byteOffset <= fileSize, + "Offset for write (%s) out of range of existing file size (%s bytes)", + byteOffset, + fileSize); synchronized (lock) { // Update metadata first. @@ -113,8 +113,8 @@ public final class DownloadDestinationOpener implements Opener<DownloadDestinati // Use ReleasableResource to ensure channel is setup properly before returning it. try (ReleasableResource<RandomAccessFile> file = - ReleasableResource.create( - fileStorage.open(onDeviceUri, RandomAccessFileOpener.createForReadWrite()))) { + ReleasableResource.create( + fileStorage.open(onDeviceUri, RandomAccessFileOpener.createForReadWrite()))) { // Get channel and seek to correct offset. FileChannel channel = file.get().getChannel(); channel.position(byteOffset); @@ -143,7 +143,7 @@ public final class DownloadDestinationOpener implements Opener<DownloadDestinati * <p>Exceptions due to an async call failure are handled and wrapped in an IOException. */ private static <V> V blockingGet(ListenableFuture<V> future, String errorMessage) - throws IOException { + throws IOException { try { return future.get(TIMEOUT_MS, MILLISECONDS); } catch (InterruptedException e) { @@ -167,17 +167,17 @@ public final class DownloadDestinationOpener implements Opener<DownloadDestinati public DownloadDestination open(OpenContext openContext) throws IOException { if (openContext.hasTransforms()) { throw new UnsupportedFileStorageOperation( - "Transforms are not supported by this Opener: " + openContext.originalUri()); + "Transforms are not supported by this Opener: " + openContext.originalUri()); } // Check whether or not the file uri is a directory. if (openContext.storage().isDirectory(openContext.originalUri())) { throw new IOException( - new IllegalArgumentException("Requested file download is already a directory.")); + new IllegalArgumentException("Requested file download is already a directory.")); } return new DownloadDestinationImpl( - openContext.originalUri(), openContext.storage(), metadataStore); + openContext.originalUri(), openContext.storage(), metadataStore); } public static DownloadDestinationOpener create(DownloadMetadataStore metadataStore) { diff --git a/java/com/google/android/libraries/mobiledatadownload/file/monitors/BUILD b/java/com/google/android/libraries/mobiledatadownload/file/monitors/BUILD index 11f0cbd..2b65923 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/monitors/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/file/monitors/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -23,7 +24,5 @@ package( android_library( name = "monitors", srcs = ["ByteCountingOutputMonitor.java"], - deps = [ - "//java/com/google/android/libraries/mobiledatadownload/file/spi", - ], + deps = ["//java/com/google/android/libraries/mobiledatadownload/file/spi"], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/AppendStreamOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/AppendStreamOpener.java index bff5543..bfb561d 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/AppendStreamOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/AppendStreamOpener.java @@ -18,6 +18,7 @@ package com.google.android.libraries.mobiledatadownload.file.openers; import com.google.android.libraries.mobiledatadownload.file.Behavior; import com.google.android.libraries.mobiledatadownload.file.OpenContext; import com.google.android.libraries.mobiledatadownload.file.Opener; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.IOException; import java.io.OutputStream; import java.util.List; @@ -37,6 +38,7 @@ public final class AppendStreamOpener implements Opener<OutputStream> { * Supports adding options to writes. For example, SyncBehavior will force data to be flushed and * durably persisted. */ + @CanIgnoreReturnValue public AppendStreamOpener withBehaviors(Behavior... behaviors) { this.behaviors = behaviors; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/BUILD b/java/com/google/android/libraries/mobiledatadownload/file/openers/BUILD index 8511d59..186c80c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -58,6 +59,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file/common", "@androidx_annotation_annotation", # buildcleaner: keep "@com_google_code_findbugs_jsr305", + "@com_google_errorprone_error_prone_annotations", ], ) @@ -69,6 +71,7 @@ android_library( ":scratch", "//java/com/google/android/libraries/mobiledatadownload/file", "@com_google_code_findbugs_jsr305", + "@com_google_errorprone_error_prone_annotations", ], ) @@ -103,6 +106,7 @@ android_library( ":scratch", ":stream", "//java/com/google/android/libraries/mobiledatadownload/file", + "@com_google_errorprone_error_prone_annotations", "@com_google_protobuf//:protobuf_lite", ], ) @@ -117,6 +121,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/file/common", "@com_google_code_findbugs_jsr305", + "@com_google_errorprone_error_prone_annotations", ], ) @@ -124,8 +129,10 @@ android_library( name = "recursive_delete", srcs = ["RecursiveDeleteOpener.java"], deps = [ + ":file", "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:exceptions", + "@com_google_errorprone_error_prone_annotations", ], ) @@ -145,7 +152,10 @@ android_library( "ReadStreamOpener.java", "WriteStreamOpener.java", ], - deps = ["//java/com/google/android/libraries/mobiledatadownload/file"], + deps = [ + "//java/com/google/android/libraries/mobiledatadownload/file", + "@com_google_errorprone_error_prone_annotations", + ], ) android_library( @@ -171,6 +181,7 @@ android_library( ":stream", "//java/com/google/android/libraries/mobiledatadownload/file", "@com_google_code_findbugs_jsr305", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], ) @@ -185,6 +196,7 @@ android_library( ":bytes", "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:charsets", + "@com_google_errorprone_error_prone_annotations", ], ) @@ -198,6 +210,7 @@ android_library( ":stream", "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/file/common", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/LockFileOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/LockFileOpener.java index c608ec2..9cfcb67 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/LockFileOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/LockFileOpener.java @@ -20,6 +20,7 @@ import com.google.android.libraries.mobiledatadownload.file.OpenContext; import com.google.android.libraries.mobiledatadownload.file.Opener; import com.google.android.libraries.mobiledatadownload.file.common.FileChannelConvertible; import com.google.android.libraries.mobiledatadownload.file.common.ReleasableResource; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.Closeable; import java.io.IOException; import java.io.RandomAccessFile; @@ -42,7 +43,7 @@ import javax.annotation.Nullable; */ public final class LockFileOpener implements Opener<Closeable> { - private static final String LOCK_SUFFIX = ".lock"; + public static final String LOCK_SUFFIX = ".lock"; private final boolean shared; private final boolean readOnly; @@ -84,6 +85,7 @@ public final class LockFileOpener implements Opener<Closeable> { * If enabled and the lock cannot be acquired immediately, {@link #open} will return {@code null} * instead of waiting until the lock can be acquired. */ + @CanIgnoreReturnValue public LockFileOpener nonBlocking(boolean isNonBlocking) { this.isNonBlocking = isNonBlocking; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/RandomAccessFileOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/RandomAccessFileOpener.java index 25e1839..6589b07 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/RandomAccessFileOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/RandomAccessFileOpener.java @@ -36,7 +36,7 @@ public final class RandomAccessFileOpener implements Opener<RandomAccessFile> { } public static RandomAccessFileOpener createForRead() { - return new RandomAccessFileOpener(/*writeSupport=*/ false); + return new RandomAccessFileOpener(/* writeSupport= */ false); } /** @@ -44,7 +44,7 @@ public final class RandomAccessFileOpener implements Opener<RandomAccessFile> { * parent directories do not exist, they will be created. */ public static RandomAccessFileOpener createForReadWrite() { - return new RandomAccessFileOpener(/*writeSupport=*/ true); + return new RandomAccessFileOpener(/* writeSupport= */ true); } @Override diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadFileOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadFileOpener.java index a02b9bb..9bb70fa 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadFileOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadFileOpener.java @@ -23,6 +23,7 @@ import com.google.android.libraries.mobiledatadownload.file.Opener; import com.google.android.libraries.mobiledatadownload.file.common.FileConvertible; import com.google.android.libraries.mobiledatadownload.file.common.ReleasableResource; import com.google.android.libraries.mobiledatadownload.file.common.UnsupportedFileStorageOperation; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -88,6 +89,7 @@ public final class ReadFileOpener implements Opener<File> { * @param context Android context for the root directory where fifos are stored. * @return This opener. */ + @CanIgnoreReturnValue public ReadFileOpener withFallbackToPipeUsingExecutor(ExecutorService executor, Context context) { this.executor = executor; this.context = context; @@ -99,6 +101,7 @@ public final class ReadFileOpener implements Opener<File> { * there are any transforms enabled. This is like the {@link UriAdapter} interface, but with more * guard rails to make it safe to expose publicly. */ + @CanIgnoreReturnValue public ReadFileOpener withShortCircuit() { this.shortCircuit = true; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadProtoOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadProtoOpener.java index 9762803..cd8530d 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadProtoOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadProtoOpener.java @@ -17,6 +17,7 @@ package com.google.android.libraries.mobiledatadownload.file.openers; import com.google.android.libraries.mobiledatadownload.file.OpenContext; import com.google.android.libraries.mobiledatadownload.file.Opener; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.protobuf.ExtensionRegistryLite; import com.google.protobuf.MessageLite; import com.google.protobuf.Parser; @@ -60,6 +61,7 @@ public final class ReadProtoOpener<T extends MessageLite> implements Opener<T> { } /** Adds an extension registry used while parsing the proto. */ + @CanIgnoreReturnValue public ReadProtoOpener<T> withExtensionRegistry(ExtensionRegistryLite registry) { this.registry = registry; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadStreamOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadStreamOpener.java index 94848ee..71dfea6 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadStreamOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadStreamOpener.java @@ -18,6 +18,7 @@ package com.google.android.libraries.mobiledatadownload.file.openers; import com.google.android.libraries.mobiledatadownload.file.Behavior; import com.google.android.libraries.mobiledatadownload.file.OpenContext; import com.google.android.libraries.mobiledatadownload.file.Opener; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; @@ -35,6 +36,7 @@ public final class ReadStreamOpener implements Opener<InputStream> { return new ReadStreamOpener(); } + @CanIgnoreReturnValue public ReadStreamOpener withBehaviors(Behavior... behaviors) { this.behaviors = behaviors; return this; @@ -48,6 +50,7 @@ public final class ReadStreamOpener implements Opener<InputStream> { * * <p>Discouraged: protos (already buffered internally). */ + @CanIgnoreReturnValue public ReadStreamOpener withBufferedIo() { this.bufferIo = true; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadStringOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadStringOpener.java index ff434aa..6f75e2c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadStringOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadStringOpener.java @@ -18,6 +18,7 @@ package com.google.android.libraries.mobiledatadownload.file.openers; import com.google.android.libraries.mobiledatadownload.file.OpenContext; import com.google.android.libraries.mobiledatadownload.file.Opener; import com.google.android.libraries.mobiledatadownload.file.common.internal.Charsets; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.IOException; import java.nio.charset.Charset; @@ -31,6 +32,7 @@ public final class ReadStringOpener implements Opener<String> { return new ReadStringOpener(); } + @CanIgnoreReturnValue public ReadStringOpener withCharset(Charset charset) { this.charset = charset; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/RecursiveDeleteOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/RecursiveDeleteOpener.java index 80fe27f..237def1 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/RecursiveDeleteOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/RecursiveDeleteOpener.java @@ -16,10 +16,17 @@ package com.google.android.libraries.mobiledatadownload.file.openers; import android.net.Uri; +import android.os.Build.VERSION; +import android.os.Build.VERSION_CODES; +import android.system.Os; +import android.system.OsConstants; +import android.system.StructStat; import com.google.android.libraries.mobiledatadownload.file.OpenContext; import com.google.android.libraries.mobiledatadownload.file.Opener; import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; import com.google.android.libraries.mobiledatadownload.file.common.internal.Exceptions; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -37,8 +44,6 @@ import java.util.List; * * <ul> * <li>Directory tree traversal is not an atomic operation - * <li>There are no special considerations for symlinks, meaning the opener could get caught in a - * recursive directory loop (i.e. a directory that contains a symlink to itself) * </ul> * * <p>Usage: <code> @@ -46,12 +51,18 @@ import java.util.List; * </code> */ public final class RecursiveDeleteOpener implements Opener<Void> { - private RecursiveDeleteOpener() {} + private boolean noFollowLinks; public static RecursiveDeleteOpener create() { return new RecursiveDeleteOpener(); } + @CanIgnoreReturnValue + public RecursiveDeleteOpener withNoFollowLinks() { + this.noFollowLinks = true; + return this; + } + @Override public Void open(OpenContext openContext) throws IOException { List<IOException> exceptions = new ArrayList<>(); @@ -63,12 +74,15 @@ public final class RecursiveDeleteOpener implements Opener<Void> { return null; // for Void return type } - private static void deleteRecursively( + private void deleteRecursively( SynchronousFileStorage storage, Uri uri, List<IOException> exceptions) { + ReadFileOpener readFileOpener = ReadFileOpener.create().withShortCircuit(); try { if (storage.isDirectory(uri)) { - for (Uri child : storage.children(uri)) { - deleteRecursively(storage, child, exceptions); + if (!noFollowLinks || !isSymlink(uri, storage, readFileOpener)) { + for (Uri child : storage.children(uri)) { + deleteRecursively(storage, child, exceptions); + } } storage.deleteDirectory(uri); } else { @@ -78,4 +92,23 @@ public final class RecursiveDeleteOpener implements Opener<Void> { exceptions.add(e); } } + + private static boolean isSymlink( + Uri uri, SynchronousFileStorage storage, ReadFileOpener readFileOpener) { + if (VERSION.SDK_INT >= VERSION_CODES.LOLLIPOP) { + try { + File file = storage.open(uri, readFileOpener); + if (file == null || !file.exists()) { + return false; + } + StructStat stat = Os.lstat(file.getAbsolutePath()); + return (stat.st_mode & OsConstants.S_IFLNK) != 0; + } catch (Exception e) { + // NOTE: this should be ErrnoException, but we're forced to catch Exception to avoid + // breaking lower sdk levels (exceptions aren't stripped from dead code blocks). + return false; + } + } + return false; + } } diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/StreamMutationOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/StreamMutationOpener.java index d26538f..d00af35 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/StreamMutationOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/StreamMutationOpener.java @@ -19,6 +19,7 @@ import android.net.Uri; import com.google.android.libraries.mobiledatadownload.file.Behavior; import com.google.android.libraries.mobiledatadownload.file.OpenContext; import com.google.android.libraries.mobiledatadownload.file.Opener; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.ByteArrayInputStream; import java.io.Closeable; import java.io.FileNotFoundException; @@ -68,12 +69,14 @@ public final class StreamMutationOpener implements Opener<StreamMutationOpener.M * Enable exclusive locking with this opener. This is useful if multiple processes or threads need * to maintain transactional isolation. */ + @CanIgnoreReturnValue public StreamMutationOpener withLocking(LockFileOpener locking) { this.locking = locking; return this; } /** Apply these behaviors while writing only. */ + @CanIgnoreReturnValue public StreamMutationOpener withBehaviors(Behavior... behaviors) { this.behaviors = behaviors; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/SystemLibraryOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/SystemLibraryOpener.java index 0f90b41..4865549 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/SystemLibraryOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/SystemLibraryOpener.java @@ -21,6 +21,7 @@ import android.util.Base64; import com.google.android.libraries.mobiledatadownload.file.OpenContext; import com.google.android.libraries.mobiledatadownload.file.Opener; import com.google.common.io.ByteStreams; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -52,6 +53,7 @@ public final class SystemLibraryOpener implements Opener<Void> { private SystemLibraryOpener() {} + @CanIgnoreReturnValue public SystemLibraryOpener withCacheDirectory(Uri dir) { this.cacheDirectory = dir; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteByteArrayOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteByteArrayOpener.java index 4676e7e..932530b 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteByteArrayOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteByteArrayOpener.java @@ -18,6 +18,7 @@ package com.google.android.libraries.mobiledatadownload.file.openers; import com.google.android.libraries.mobiledatadownload.file.Behavior; import com.google.android.libraries.mobiledatadownload.file.OpenContext; import com.google.android.libraries.mobiledatadownload.file.Opener; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.IOException; import java.io.OutputStream; @@ -37,6 +38,7 @@ public final class WriteByteArrayOpener implements Opener<Void> { this.bytesToWrite = bytesToWrite; } + @CanIgnoreReturnValue public WriteByteArrayOpener withBehaviors(Behavior... behaviors) { this.behaviors = behaviors; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteFileOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteFileOpener.java index c930f11..2b49af4 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteFileOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteFileOpener.java @@ -22,6 +22,7 @@ import com.google.android.libraries.mobiledatadownload.file.Opener; import com.google.android.libraries.mobiledatadownload.file.common.FileConvertible; import com.google.android.libraries.mobiledatadownload.file.common.ReleasableResource; import com.google.android.libraries.mobiledatadownload.file.openers.WriteFileOpener.FileCloser; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.Closeable; import java.io.File; import java.io.FileInputStream; @@ -148,6 +149,7 @@ public final class WriteFileOpener implements Opener<FileCloser> { * @param context Android context for the root directory where fifos are stored. * @return This opener. */ + @CanIgnoreReturnValue public WriteFileOpener withFallbackToPipeUsingExecutor( ExecutorService executor, Context context) { this.executor = executor; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteProtoOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteProtoOpener.java index 81f3eb6..4431ffa 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteProtoOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteProtoOpener.java @@ -19,6 +19,7 @@ import android.net.Uri; import com.google.android.libraries.mobiledatadownload.file.Behavior; import com.google.android.libraries.mobiledatadownload.file.OpenContext; import com.google.android.libraries.mobiledatadownload.file.Opener; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.protobuf.MessageLite; import java.io.FileNotFoundException; import java.io.IOException; @@ -43,6 +44,7 @@ public final class WriteProtoOpener implements Opener<Void> { * Supports adding options to writes. For example, SyncBehavior will force data to be flushed and * durably persisted. */ + @CanIgnoreReturnValue public WriteProtoOpener withBehaviors(Behavior... behaviors) { this.behaviors = behaviors; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteStreamOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteStreamOpener.java index f6e6c37..3eccfdd 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteStreamOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteStreamOpener.java @@ -18,6 +18,7 @@ package com.google.android.libraries.mobiledatadownload.file.openers; import com.google.android.libraries.mobiledatadownload.file.Behavior; import com.google.android.libraries.mobiledatadownload.file.OpenContext; import com.google.android.libraries.mobiledatadownload.file.Opener; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.IOException; import java.io.OutputStream; import java.util.List; @@ -35,6 +36,7 @@ public final class WriteStreamOpener implements Opener<OutputStream> { return new WriteStreamOpener(); } + @CanIgnoreReturnValue public WriteStreamOpener withBehaviors(Behavior... behaviors) { this.behaviors = behaviors; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteStringOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteStringOpener.java index 9c2a98c..2c8fd8f 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteStringOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteStringOpener.java @@ -19,6 +19,7 @@ import com.google.android.libraries.mobiledatadownload.file.Behavior; import com.google.android.libraries.mobiledatadownload.file.OpenContext; import com.google.android.libraries.mobiledatadownload.file.Opener; import com.google.android.libraries.mobiledatadownload.file.common.internal.Charsets; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.IOException; import java.nio.charset.Charset; @@ -36,11 +37,13 @@ public final class WriteStringOpener implements Opener<Void> { return new WriteStringOpener(string); } + @CanIgnoreReturnValue public WriteStringOpener withCharset(Charset charset) { this.charset = charset; return this; } + @CanIgnoreReturnValue public WriteStringOpener withBehaviors(Behavior... behaviors) { this.behaviors = behaviors; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/samples/BUILD b/java/com/google/android/libraries/mobiledatadownload/file/samples/BUILD index 0d21230..f5a8afb 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/samples/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/file/samples/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -27,9 +28,11 @@ android_library( deps = [ "//java/com/google/android/libraries/mobiledatadownload/file/backends:file", "//java/com/google/android/libraries/mobiledatadownload/file/common", + "//java/com/google/android/libraries/mobiledatadownload/file/common:fragment", "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:charsets", "//java/com/google/android/libraries/mobiledatadownload/file/spi", "@androidx_appcompat_appcompat", # buildcleaner: keep + "@com_google_code_findbugs_jsr305", "@com_google_guava_guava", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/file/samples/CapitalizationTransform.java b/java/com/google/android/libraries/mobiledatadownload/file/samples/CapitalizationTransform.java index 73d99d8..8a55656 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/samples/CapitalizationTransform.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/samples/CapitalizationTransform.java @@ -21,6 +21,7 @@ import com.google.android.libraries.mobiledatadownload.file.spi.Transform; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import javax.annotation.Nullable; /** * This is a toy transform that is useful to illustrate that the invocation order is correct when @@ -59,7 +60,7 @@ public final class CapitalizationTransform implements Transform { } @Override - public Long size() throws IOException { + public @Nullable Long size() throws IOException { if (!(in instanceof Sizable)) { return null; } diff --git a/java/com/google/android/libraries/mobiledatadownload/file/spi/BUILD b/java/com/google/android/libraries/mobiledatadownload/file/spi/BUILD index 23cc319..49458c1 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/spi/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/file/spi/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -25,6 +26,7 @@ android_library( srcs = glob(["*.java"]), deps = [ "//java/com/google/android/libraries/mobiledatadownload/file/common", + "@com_google_errorprone_error_prone_annotations", "@com_google_code_findbugs_jsr305", # NOTE: dependency of gmscore client lib <internal> ], diff --git a/java/com/google/android/libraries/mobiledatadownload/file/transforms/BUILD b/java/com/google/android/libraries/mobiledatadownload/file/transforms/BUILD index 9f9525d..1933092 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/transforms/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/file/transforms/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -65,6 +66,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:charsets", "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:lite_transform_fragments", "//proto:transform_java_proto_lite", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/foreground/BUILD b/java/com/google/android/libraries/mobiledatadownload/foreground/BUILD index b98c623..8a120c9 100644 --- a/java/com/google/android/libraries/mobiledatadownload/foreground/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/foreground/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -27,6 +28,18 @@ filegroup( ]), ) +android_library( + name = "ForegroundDownloadKey", + srcs = ["ForegroundDownloadKey.java"], + deps = [ + "//java/com/google/android/libraries/mobiledatadownload/account:AccountUtil", + "//java/com/google/android/libraries/mobiledatadownload/internal:MddConstants", + "//third_party/java/android_libs/guava_jdk5:hash", + "@com_google_auto_value", + "@com_google_guava_guava", + ], +) + # This includes all translated strings for MDD Notifications. Apps can choose to include subset of the # supported locale resources in their binary using the `resource_configuration_filters` option in # their android_binary rule. For more info, see: <internal> @@ -34,7 +47,8 @@ android_library( name = "NotificationUtil", srcs = ["NotificationUtil.java"], manifest = "AndroidManifest.xml", - resource_files = glob(["res/**"]), + resource_files = glob(["res/**"]) + [ + ], deps = [ "@androidx_annotation_annotation", "@androidx_core_core", diff --git a/java/com/google/android/libraries/mobiledatadownload/foreground/ForegroundDownloadKey.java b/java/com/google/android/libraries/mobiledatadownload/foreground/ForegroundDownloadKey.java new file mode 100644 index 0000000..a4f3ea2 --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/foreground/ForegroundDownloadKey.java @@ -0,0 +1,100 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.foreground; + +import static com.google.android.libraries.mobiledatadownload.internal.MddConstants.SPLIT_CHAR; + +import android.accounts.Account; +import android.net.Uri; +import com.google.android.libraries.mobiledatadownload.account.AccountUtil; +import com.google.auto.value.AutoValue; +import com.google.common.base.Optional; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; + +/** + * Container class for unique key of a foreground download. + * + * <p>There are two kinds of foreground downloads supported: file group and single files. + * + * <p>Each kind has different requirements to build the unique key that must be provided when + * building a ForegroundDownloadKey. + */ +@AutoValue +public abstract class ForegroundDownloadKey { + + /** + * Kind of {@link ForegroundDownloadKey}. + * + * <p>Only two types of foreground downloads are supported, file groups and single files. + */ + public enum Kind { + FILE_GROUP, + SINGLE_FILE, + } + + public abstract Kind kind(); + + public abstract String key(); + + /** + * Unique Identifier of a File Group used to identify a group during a foreground download. + * + * <p><b>NOTE:</b> Properties set here <em>must</em> match the properties set in {@link + * DownloadFileGroupRequest} when starting a Foreground Download. + * + * @param groupName The name of the group to download (required) + * @param account An associated account of the group, if applicable (optional) + * @param variantId An associated variantId fo the group, if applicable (optional) + */ + public static ForegroundDownloadKey ofFileGroup( + String groupName, Optional<Account> account, Optional<String> variantId) { + Hasher keyHasher = Hashing.sha256().newHasher().putUnencodedChars(groupName); + + if (account.isPresent()) { + keyHasher + .putUnencodedChars(SPLIT_CHAR) + .putUnencodedChars(AccountUtil.serialize(account.get())); + } + + if (variantId.isPresent()) { + keyHasher.putUnencodedChars(SPLIT_CHAR).putUnencodedChars(variantId.get()); + } + return new AutoValue_ForegroundDownloadKey(Kind.FILE_GROUP, keyHasher.hash().toString()); + } + + /** + * Unique Identifier of a File used to identify it during a foreground download. + * + * <p><b>NOTE:</b> Properties set here <em>must</em> match the properties set in {@link + * SingleFileDownloadRequest} or {@link DownloadRequest} when starting a Foreground Download. + * + * @param destinationUri The on-device location where the file will be downloaded (required) + */ + public static ForegroundDownloadKey ofSingleFile(Uri destinationUri) { + Hasher keyHasher = + Hashing.sha256() + .newHasher() + .putUnencodedChars(destinationUri.toString()) + .putUnencodedChars(SPLIT_CHAR); + return new AutoValue_ForegroundDownloadKey(Kind.SINGLE_FILE, keyHasher.hash().toString()); + } + + @Override + public final String toString() { + return key(); + } +} diff --git a/java/com/google/android/libraries/mobiledatadownload/foreground/NotificationUtil.java b/java/com/google/android/libraries/mobiledatadownload/foreground/NotificationUtil.java index 5ba9a90..75ddfc8 100644 --- a/java/com/google/android/libraries/mobiledatadownload/foreground/NotificationUtil.java +++ b/java/com/google/android/libraries/mobiledatadownload/foreground/NotificationUtil.java @@ -22,178 +22,192 @@ import android.content.Context; import android.content.Intent; import android.os.Build.VERSION; import android.os.Build.VERSION_CODES; + import androidx.annotation.RequiresApi; import androidx.core.app.NotificationCompat; -import androidx.core.app.NotificationCompat.BigTextStyle; import androidx.core.app.NotificationManagerCompat; import androidx.core.content.ContextCompat; + import com.google.common.base.Preconditions; + import javax.annotation.Nullable; /** Utilities for creating and managing notifications. */ // TODO(b/148401016): Add UI test for NotificationUtil. public final class NotificationUtil { - public static final String CANCEL_ACTION_EXTRA = "cancel-action"; - public static final String KEY_EXTRA = "key"; - public static final String STOP_SERVICE_EXTRA = "stop-service"; - - private NotificationUtil() {} - - public static final String NOTIFICATION_CHANNEL_ID = "download-notification-channel-id"; - - /** Create the NotificationBuilder for the Foreground Download Service */ - public static NotificationCompat.Builder createForegroundServiceNotificationBuilder( - Context context) { - return getNotificationBuilder(context) - .setContentTitle( - "Downloading") - .setSmallIcon(android.R.drawable.stat_notify_sync_noanim); - } - - /** Create a Notification.Builder. */ - public static NotificationCompat.Builder createNotificationBuilder( - Context context, int size, String contentTitle, String contentText) { - return getNotificationBuilder(context) - .setContentTitle(contentTitle) - .setContentText(contentText) - .setSmallIcon(android.R.drawable.stat_sys_download) - .setOngoing(true) - .setProgress(size, 0, false) - .setStyle(new BigTextStyle().bigText(contentText)); - } - - private static NotificationCompat.Builder getNotificationBuilder(Context context) { - return new NotificationCompat.Builder(context, NOTIFICATION_CHANNEL_ID) - .setCategory(NotificationCompat.CATEGORY_SERVICE) - .setOnlyAlertOnce(true); - } - - /** - * Create a Notification for a key. - * - * @param key Key to identify the download this notification is created for. - */ - public static void cancelNotificationForKey(Context context, String key) { - NotificationManagerCompat notificationManager = NotificationManagerCompat.from(context); - notificationManager.cancel(notificationKeyForKey(key)); - } - - /** Create the Cancel Menu Action which will be attach to the download notification. */ - // FLAG_IMMUTABLE is only for api >= 23, however framework still recommends to use this: - // <internal> - @SuppressLint("InlinedApi") - public static void createCancelAction( - Context context, - Class<?> foregroundDownloadServiceClass, - String key, - NotificationCompat.Builder notification, - int notificationKey) { - SaferIntentUtils intentUtils = new SaferIntentUtils() {}; - - Intent cancelIntent = new Intent(context, foregroundDownloadServiceClass); - cancelIntent.setPackage(context.getPackageName()); - cancelIntent.putExtra(CANCEL_ACTION_EXTRA, notificationKey); - cancelIntent.putExtra(KEY_EXTRA, key); - - // It should be safe since we are using SaferPendingIntent, setting Package and Component, and - // use PendingIntent.FLAG_ONE_SHOT | PendingIntent.FLAG_IMMUTABLE. - PendingIntent pendingCancelIntent; - if (VERSION.SDK_INT >= VERSION_CODES.O) { - pendingCancelIntent = - intentUtils.getForegroundService( - context, - notificationKey, - cancelIntent, - PendingIntent.FLAG_ONE_SHOT | PendingIntent.FLAG_IMMUTABLE); - } else { - pendingCancelIntent = - intentUtils.getService( - context, - notificationKey, - cancelIntent, - PendingIntent.FLAG_ONE_SHOT | PendingIntent.FLAG_IMMUTABLE); + public static final String CANCEL_ACTION_EXTRA = "cancel-action"; + public static final String KEY_EXTRA = "key"; + public static final String STOP_SERVICE_EXTRA = "stop-service"; + + private NotificationUtil() { + } + + public static final String NOTIFICATION_CHANNEL_ID = "download-notification-channel-id"; + + /** Create the NotificationBuilder for the Foreground Download Service */ + public static NotificationCompat.Builder createForegroundServiceNotificationBuilder( + Context context) { + return getNotificationBuilder(context) + .setContentTitle("Downloading") + .setSmallIcon(android.R.drawable.stat_notify_sync_noanim); + } + + /** Create a Notification.Builder. */ + public static NotificationCompat.Builder createNotificationBuilder( + Context context, int size, String contentTitle, String contentText) { + return getNotificationBuilder(context) + .setContentTitle(contentTitle) + .setContentText(contentText) + .setSmallIcon(android.R.drawable.stat_sys_download) + .setOngoing(true) + .setProgress(size, 0, false); + } + + private static NotificationCompat.Builder getNotificationBuilder(Context context) { + return new NotificationCompat.Builder(context, NOTIFICATION_CHANNEL_ID) + .setCategory(NotificationCompat.CATEGORY_SERVICE) + .setOnlyAlertOnce(true); + } + + /** + * Create a Notification for a key. + * + * @param key Key to identify the download this notification is created for. + */ + public static void cancelNotificationForKey(Context context, String key) { + NotificationManagerCompat notificationManager = NotificationManagerCompat.from(context); + notificationManager.cancel(notificationKeyForKey(key)); + } + + /** Create the Cancel Menu Action which will be attach to the download notification. */ + // FLAG_IMMUTABLE is only for api >= 23, however framework still recommends to use this: + // <internal> + @SuppressLint("InlinedApi") + public static void createCancelAction( + Context context, + Class<?> foregroundDownloadServiceClass, + String key, + NotificationCompat.Builder notification, + int notificationKey) { + SaferIntentUtils intentUtils = new SaferIntentUtils() { + }; + + Intent cancelIntent = new Intent(context, foregroundDownloadServiceClass); + cancelIntent.setPackage(context.getPackageName()); + cancelIntent.putExtra(CANCEL_ACTION_EXTRA, notificationKey); + cancelIntent.putExtra(KEY_EXTRA, key); + + // It should be safe since we are using SaferPendingIntent, setting Package and + // Component, and + // use PendingIntent.FLAG_ONE_SHOT | PendingIntent.FLAG_IMMUTABLE. + PendingIntent pendingCancelIntent; + if (VERSION.SDK_INT >= VERSION_CODES.O) { + pendingCancelIntent = + intentUtils.getForegroundService( + context, + notificationKey, + cancelIntent, + PendingIntent.FLAG_ONE_SHOT | PendingIntent.FLAG_IMMUTABLE); + } else { + pendingCancelIntent = + intentUtils.getService( + context, + notificationKey, + cancelIntent, + PendingIntent.FLAG_ONE_SHOT | PendingIntent.FLAG_IMMUTABLE); + } + NotificationCompat.Action action = + new NotificationCompat.Action.Builder( + android.R.drawable.stat_sys_warning, + "Cancel", + Preconditions.checkNotNull(pendingCancelIntent)) + .build(); + notification.addAction(action); } - NotificationCompat.Action action = - new NotificationCompat.Action.Builder( - android.R.drawable.stat_sys_warning, - "Cancel", - Preconditions.checkNotNull(pendingCancelIntent)) - .build(); - notification.addAction(action); - } - - /** Generate the Notification Key for the Key */ - public static int notificationKeyForKey(String key) { - // Consider if we could have collision. - // Heavier alternative is Hashing.goodFastHash(32).hashUnencodedChars(key).asInt(); - return key.hashCode(); - } - - /** Send intent to start the DownloadService in foreground. */ - public static void startForegroundDownloadService( - Context context, Class<?> foregroundDownloadService, String key) { - Intent intent = new Intent(context, foregroundDownloadService); - intent.putExtra(KEY_EXTRA, key); - - // Start ForegroundDownloadService to download in the foreground. - ContextCompat.startForegroundService(context, intent); - } - - /** Sending the intent to stop the foreground download service */ - public static void stopForegroundDownloadService( - Context context, Class<?> foregroundDownloadService) { - Intent intent = new Intent(context, foregroundDownloadService); - intent.putExtra(STOP_SERVICE_EXTRA, true); - - // This will send the intent to stop the service. - ContextCompat.startForegroundService(context, intent); - } - - /** - * Return the String message to display in Notification when the download is paused to wait for - * network connection. - */ - public static String getDownloadPausedMessage(Context context) { - return "Waiting for network connection"; - } - - /** Return the String message to display in Notification when the download is failed. */ - public static String getDownloadFailedMessage(Context context) { - return "Download failed"; - } - - /** Return the String message to display in Notification when the download is success. */ - public static String getDownloadSuccessMessage(Context context) { - return "Downloaded"; - } - - /** Create the Notification Channel for Downloading. */ - public static void createNotificationChannel(Context context) { - if (VERSION.SDK_INT >= VERSION_CODES.O) { - NotificationChannel notificationChannel = - new NotificationChannel( - NOTIFICATION_CHANNEL_ID, - "Data Download Notification Channel", - android.app.NotificationManager.IMPORTANCE_DEFAULT); - - android.app.NotificationManager manager = - context.getSystemService(android.app.NotificationManager.class); - manager.createNotificationChannel(notificationChannel); + + /** Generate the Notification Key for the Key */ + public static int notificationKeyForKey(String key) { + // Consider if we could have collision. + // Heavier alternative is Hashing.goodFastHash(32).hashUnencodedChars(key).asInt(); + return key.hashCode(); } - } - - /** Utilities for safely accessing PendingIntent APIs. */ - private interface SaferIntentUtils { - @Nullable - @RequiresApi(VERSION_CODES.O) // to match PendingIntent.getForegroundService() - default PendingIntent getForegroundService( - Context context, int requestCode, Intent intent, int flags) { - return PendingIntent.getForegroundService(context, requestCode, intent, flags); + + /** Send intent to start the DownloadService in foreground. */ + public static void startForegroundDownloadService( + Context context, Class<?> foregroundDownloadService, String key) { + Intent intent = new Intent(context, foregroundDownloadService); + intent.putExtra(KEY_EXTRA, key); + + // Start ForegroundDownloadService to download in the foreground. + ContextCompat.startForegroundService(context, intent); + } + + /** Sending the intent to stop the foreground download service */ + public static void stopForegroundDownloadService( + Context context, Class<?> foregroundDownloadService, String key) { + Intent intent = new Intent(context, foregroundDownloadService); + intent.putExtra(STOP_SERVICE_EXTRA, true); + intent.putExtra(KEY_EXTRA, key); + + // This will send the intent to stop the service. + ContextCompat.startForegroundService(context, intent); + } + + /** + * Return the String message to display in Notification when the download is paused to wait for + * network connection. + */ + public static String getDownloadPausedMessage(Context context) { + return "Waiting for network connection"; + } + + /** + * Return the String message to display in Notification when the download is paused due to a + * missing wifi connection. + */ + public static String getDownloadPausedWifiMessage(Context context) { + return "Waiting for WiFi connection"; + } + + /** Return the String message to display in Notification when the download is failed. */ + public static String getDownloadFailedMessage(Context context) { + return "Download failed"; + } + + /** Return the String message to display in Notification when the download is success. */ + public static String getDownloadSuccessMessage(Context context) { + return "Downloaded"; + } + + /** Create the Notification Channel for Downloading. */ + public static void createNotificationChannel(Context context) { + if (VERSION.SDK_INT >= VERSION_CODES.O) { + NotificationChannel notificationChannel = + new NotificationChannel( + NOTIFICATION_CHANNEL_ID, + "Data Download Notification Channel", + android.app.NotificationManager.IMPORTANCE_DEFAULT); + + android.app.NotificationManager manager = + context.getSystemService(android.app.NotificationManager.class); + manager.createNotificationChannel(notificationChannel); + } } - @Nullable - default PendingIntent getService(Context context, int requestCode, Intent intent, int flags) { - return PendingIntent.getService(context, requestCode, intent, flags); + /** Utilities for safely accessing PendingIntent APIs. */ + private interface SaferIntentUtils { + + @Nullable + @RequiresApi(VERSION_CODES.O) // to match PendingIntent.getForegroundService() + default PendingIntent getForegroundService( + Context context, int requestCode, Intent intent, int flags) { + return PendingIntent.getForegroundService(context, requestCode, intent, flags); + } + + @Nullable + default PendingIntent getService(Context context, int requestCode, Intent intent, + int flags) { + return PendingIntent.getService(context, requestCode, intent, flags); + } } - } } diff --git a/java/com/google/android/libraries/mobiledatadownload/foreground/res/values/strings.xml b/java/com/google/android/libraries/mobiledatadownload/foreground/res/values/strings.xml index 1896b0c..1b7fb7a 100644 --- a/java/com/google/android/libraries/mobiledatadownload/foreground/res/values/strings.xml +++ b/java/com/google/android/libraries/mobiledatadownload/foreground/res/values/strings.xml @@ -30,11 +30,17 @@ </string> <!-- Notification title that is shown for every file that is currently - downloading but is temporary paused due to network connection. [CHAR_LIMIT=80] --> + downloading but is temporary paused due to missing any network connection. [CHAR_LIMIT=80] --> <string name="mdd_notification_download_paused"> Waiting for network connection </string> + <!-- Notification title that is shown for every file that is currently + downloading but is temporary paused due to missing wifi network connection. [CHAR_LIMIT=80] --> + <string name="mdd_notification_download_paused_wifi"> + Waiting for WiFi connection + </string> + <!-- Notification title that is shown for every file that was successfully downloaded.[CHAR_LIMIT=80] --> <string name="mdd_notification_download_success"> diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/AndroidTimeSource.java b/java/com/google/android/libraries/mobiledatadownload/internal/AndroidTimeSource.java new file mode 100644 index 0000000..71984cb --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/internal/AndroidTimeSource.java @@ -0,0 +1,41 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.internal; + +import android.os.Build.VERSION_CODES; +import android.os.SystemClock; +import androidx.annotation.RequiresApi; +import com.google.android.libraries.mobiledatadownload.TimeSource; + +/** + * Implementation of {@link com.google.android.libraries.mobiledatadownload.TimeSource} based on + * Android platform APIs. + */ + +// necessary since cgal.clock isn't available in 3P +@RequiresApi(VERSION_CODES.JELLY_BEAN_MR1) // android.os.SystemClock#elapsedRealtimeNanos +public final class AndroidTimeSource implements TimeSource { + + @Override + public long currentTimeMillis() { + return System.currentTimeMillis(); + } + + @Override + public long elapsedRealtimeNanos() { + return SystemClock.elapsedRealtimeNanos(); + } +} diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/BUILD b/java/com/google/android/libraries/mobiledatadownload/internal/BUILD index 68f9139..a0fbf4d 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/internal/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -37,8 +38,10 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/annotations", "//java/com/google/android/libraries/mobiledatadownload/file/transforms:proto", "//java/com/google/android/libraries/mobiledatadownload/internal/annotations:SequentialControlExecutor", + "//java/com/google/android/libraries/mobiledatadownload/internal/collect", "//java/com/google/android/libraries/mobiledatadownload/internal/downloader:FileValidator", "//java/com/google/android/libraries/mobiledatadownload/internal/experimentation:DownloadStageManager", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:DownloadStateLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:FileGroupStatsLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", @@ -49,6 +52,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/internal/util:FileGroupUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/util:SharedPreferencesUtil", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "//proto:log_enums_java_proto_lite", "//proto:transform_java_proto_lite", "@androidx_annotation_annotation", "@com_google_code_findbugs_jsr305", @@ -99,6 +103,7 @@ android_library( deps = [ "//java/com/google/android/libraries/mobiledatadownload:SilentFeedback", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", + "@com_google_errorprone_error_prone_annotations", ], ) @@ -134,6 +139,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/annotations", "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/internal/annotations:SequentialControlExecutor", + "//java/com/google/android/libraries/mobiledatadownload/internal/collect", "//java/com/google/android/libraries/mobiledatadownload/internal/experimentation:DownloadStageManager", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:DownloadStateLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", @@ -145,6 +151,9 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/internal/util:SymlinkUtil", "//java/com/google/android/libraries/mobiledatadownload/tracing", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", + "//proto:transform_java_proto_lite", "@androidx_annotation_annotation", "@com_google_auto_value", "@com_google_code_findbugs_jsr305", @@ -159,6 +168,7 @@ android_library( name = "FileGroupsMetadata", srcs = ["FileGroupsMetadata.java"], deps = [ + "//java/com/google/android/libraries/mobiledatadownload/internal/collect", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "@com_google_guava_guava", "@org_checkerframework_qual", @@ -175,12 +185,14 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload:TimeSource", "//java/com/google/android/libraries/mobiledatadownload/annotations", "//java/com/google/android/libraries/mobiledatadownload/internal/annotations:SequentialControlExecutor", + "//java/com/google/android/libraries/mobiledatadownload/internal/collect", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//java/com/google/android/libraries/mobiledatadownload/internal/util:FileGroupUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/util:FileGroupsMetadataUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/util:ProtoLiteUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/util:SharedPreferencesUtil", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "@androidx_annotation_annotation", "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", @@ -203,12 +215,15 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/annotations", "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/internal/annotations:SequentialControlExecutor", + "//java/com/google/android/libraries/mobiledatadownload/internal/collect", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//java/com/google/android/libraries/mobiledatadownload/internal/util:DirectoryUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/util:FileGroupUtil", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "@androidx_annotation_annotation", "@com_google_guava_guava", "@javax_inject", @@ -245,6 +260,9 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/internal/util:DirectoryUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/util:SharedPreferencesUtil", "//java/com/google/android/libraries/mobiledatadownload/monitor:DownloadProgressMonitor", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "@androidx_annotation_annotation", "@com_google_code_findbugs_jsr305", "@com_google_dagger", @@ -283,10 +301,41 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//java/com/google/android/libraries/mobiledatadownload/internal/util:SharedFilesMetadataUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/util:SharedPreferencesUtil", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "//proto:transform_java_proto_lite", "@androidx_annotation_annotation", "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", "@javax_inject", + "@org_checkerframework_qual", + ], +) + +android_library( + name = "DownloadGroupState", + srcs = ["DownloadGroupState.java"], + deps = [ + "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "//proto:client_config_java_proto_lite", + "@com_google_code_findbugs_jsr305", + "@com_google_guava_guava", + ], +) + +android_library( + name = "AndroidTimeSource", + srcs = ["AndroidTimeSource.java"], + deps = [ + "//java/com/google/android/libraries/mobiledatadownload:TimeSource", + "@androidx_annotation_annotation", + ], +) + +android_library( + name = "ExceptionToMddResultMapper", + srcs = ["ExceptionToMddResultMapper.java"], + deps = [ + "//java/com/google/android/libraries/mobiledatadownload:DownloadException", + "//proto:log_enums_java_proto_lite", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/DataFileGroupValidator.java b/java/com/google/android/libraries/mobiledatadownload/internal/DataFileGroupValidator.java index 3d49157..f05e831 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/DataFileGroupValidator.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/DataFileGroupValidator.java @@ -20,7 +20,6 @@ import com.google.android.libraries.mobiledatadownload.Flags; import com.google.android.libraries.mobiledatadownload.file.transforms.TransformProtos; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupUtil; -import com.google.mobiledatadownload.TransformProto.Transforms; import com.google.mobiledatadownload.internal.MetadataProto.DataFile; import com.google.mobiledatadownload.internal.MetadataProto.DataFile.ChecksumType; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; @@ -28,6 +27,7 @@ import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInterna import com.google.mobiledatadownload.internal.MetadataProto.DeltaFile; import com.google.mobiledatadownload.internal.MetadataProto.DeltaFile.DiffDecoder; import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions.DeviceNetworkPolicy; +import com.google.mobiledatadownload.TransformProto.Transforms; /** DataFileGroupValidator - validates the passed in DataFileGroup */ public class DataFileGroupValidator { diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/DownloadGroupState.java b/java/com/google/android/libraries/mobiledatadownload/internal/DownloadGroupState.java new file mode 100644 index 0000000..1833f6f --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/internal/DownloadGroupState.java @@ -0,0 +1,183 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.internal; + +import com.google.common.util.concurrent.ListenableFuture; +import com.google.mobiledatadownload.ClientConfigProto.ClientFileGroup; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +/** A helper class that includes information about the state of a file group download. */ +@Immutable +public abstract class DownloadGroupState { + /** The kind of {@link DownloadGroupState}. */ + public enum Kind { + /** A pending that hasn't been downloaded yet. */ + PENDING_GROUP, + + /** A pending group whose download has already stated. */ + IN_PROGRESS_FUTURE, + + /** A group that has already been downloaded. */ + DOWNLOADED_GROUP, + } + + public abstract Kind getKind(); + + public abstract DataFileGroupInternal pendingGroup(); + + public abstract ListenableFuture<ClientFileGroup> inProgressFuture(); + + public abstract ClientFileGroup downloadedGroup(); + + public static DownloadGroupState ofPendingGroup(DataFileGroupInternal dataFileGroup) { + return new ImplPendingGroup(dataFileGroup); + } + + public static DownloadGroupState ofInProgressFuture( + ListenableFuture<ClientFileGroup> clientFileGroupFuture) { + return new ImplInProgressFuture(clientFileGroupFuture); + } + + public static DownloadGroupState ofDownloadedGroup(ClientFileGroup clientFileGroup) { + return new ImplDownloadedGroup(clientFileGroup); + } + + private DownloadGroupState() {} + + // Parent class that each implementation will inherit from. + private abstract static class Parent extends DownloadGroupState { + @Override + public DataFileGroupInternal pendingGroup() { + throw new UnsupportedOperationException(getKind().toString()); + } + + @Override + public ListenableFuture<ClientFileGroup> inProgressFuture() { + throw new UnsupportedOperationException(getKind().toString()); + } + + @Override + public ClientFileGroup downloadedGroup() { + throw new UnsupportedOperationException(getKind().toString()); + } + } + + // Implementation when the contained property is "pendingGroup". + private static final class ImplPendingGroup extends Parent { + private final DataFileGroupInternal pendingGroup; + + ImplPendingGroup(DataFileGroupInternal pendingGroup) { + this.pendingGroup = pendingGroup; + } + + @Override + public DataFileGroupInternal pendingGroup() { + return pendingGroup; + } + + @Override + public DownloadGroupState.Kind getKind() { + return DownloadGroupState.Kind.PENDING_GROUP; + } + + @Override + public boolean equals(@Nullable Object x) { + if (x instanceof DownloadGroupState) { + DownloadGroupState that = (DownloadGroupState) x; + return this.getKind() == that.getKind() && this.pendingGroup.equals(that.pendingGroup()); + } else { + return false; + } + } + + @Override + public int hashCode() { + return pendingGroup.hashCode(); + } + } + + // Implementation when the contained property is "inProgressFuture". + private static final class ImplInProgressFuture extends Parent { + private final ListenableFuture<ClientFileGroup> inProgressFuture; + + ImplInProgressFuture(ListenableFuture<ClientFileGroup> inProgressFuture) { + this.inProgressFuture = inProgressFuture; + } + + @Override + public ListenableFuture<ClientFileGroup> inProgressFuture() { + return inProgressFuture; + } + + @Override + public DownloadGroupState.Kind getKind() { + return DownloadGroupState.Kind.IN_PROGRESS_FUTURE; + } + + @Override + public boolean equals(@Nullable Object x) { + if (x instanceof DownloadGroupState) { + DownloadGroupState that = (DownloadGroupState) x; + return this.getKind() == that.getKind() + && this.inProgressFuture.equals(that.inProgressFuture()); + } else { + return false; + } + } + + @Override + public int hashCode() { + return inProgressFuture.hashCode(); + } + } + + // Implementation when the contained property is "downloadedGroup". + private static final class ImplDownloadedGroup extends Parent { + private final ClientFileGroup downloadedGroup; + + ImplDownloadedGroup(ClientFileGroup downloadedGroup) { + this.downloadedGroup = downloadedGroup; + } + + @Override + public ClientFileGroup downloadedGroup() { + return downloadedGroup; + } + + @Override + public DownloadGroupState.Kind getKind() { + return DownloadGroupState.Kind.DOWNLOADED_GROUP; + } + + @Override + public boolean equals(@Nullable Object x) { + if (x instanceof DownloadGroupState) { + DownloadGroupState that = (DownloadGroupState) x; + return this.getKind() == that.getKind() + && this.downloadedGroup.equals(that.downloadedGroup()); + } else { + return false; + } + } + + @Override + public int hashCode() { + return downloadedGroup.hashCode(); + } + } +} diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/ExceptionToMddResultMapper.java b/java/com/google/android/libraries/mobiledatadownload/internal/ExceptionToMddResultMapper.java new file mode 100644 index 0000000..f5fa536 --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/internal/ExceptionToMddResultMapper.java @@ -0,0 +1,65 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.internal; + +import com.google.android.libraries.mobiledatadownload.DownloadException; +import java.io.IOException; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutionException; + +/** + * Maps exception to MddLibApiResult.Code. Used for logging. + * + * @see wireless.android.icing.proto.MddLibApiResult + */ +public final class ExceptionToMddResultMapper { + + private ExceptionToMddResultMapper() {} + + /** + * Maps Exception to appropriate int for logging. + * + * <p>If t is an ExecutionException, then the cause (t.getCause()) is mapped. + */ + public static int map(Throwable t) { + + Throwable cause; + if (t instanceof ExecutionException) { + cause = t.getCause(); + } else { + cause = t; + } + + if (cause instanceof CancellationException) { + return 0; + } else if (cause instanceof InterruptedException) { + return 0; + } else if (cause instanceof IOException) { + return 0; + } else if (cause instanceof IllegalStateException) { + return 0; + } else if (cause instanceof IllegalArgumentException) { + return 0; + } else if (cause instanceof UnsupportedOperationException) { + return 0; + } else if (cause instanceof DownloadException) { + return 0; + } + + // Capturing all other errors occurred during execution as unknown errors. + return 0; + } +} diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/ExpirationHandler.java b/java/com/google/android/libraries/mobiledatadownload/internal/ExpirationHandler.java index b5efe22..7c0f698 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/ExpirationHandler.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/ExpirationHandler.java @@ -20,7 +20,6 @@ import static java.lang.Math.min; import android.content.Context; import android.net.Uri; -import android.util.Pair; import androidx.annotation.VisibleForTesting; import com.google.android.libraries.mobiledatadownload.Flags; import com.google.android.libraries.mobiledatadownload.SilentFeedback; @@ -28,6 +27,7 @@ import com.google.android.libraries.mobiledatadownload.TimeSource; import com.google.android.libraries.mobiledatadownload.annotations.InstanceId; import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; import com.google.android.libraries.mobiledatadownload.internal.annotations.SequentialControlExecutor; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup; import com.google.android.libraries.mobiledatadownload.internal.logging.EventLogger; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.android.libraries.mobiledatadownload.internal.util.DirectoryUtil; @@ -39,6 +39,7 @@ import com.google.mobiledatadownload.internal.MetadataProto.DataFile; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; import java.io.IOException; import java.util.ArrayList; import java.util.HashSet; @@ -98,8 +99,7 @@ public class ExpirationHandler { this.flags = flags; } - // TODO(b/124072754): Change to package private once all code is refactored. - public ListenableFuture<Void> updateExpiration() { + ListenableFuture<Void> updateExpiration() { return PropagatedFutures.transformAsync( removeExpiredStaleGroups(), voidArg0 -> @@ -116,16 +116,16 @@ public class ExpirationHandler { fileGroupsMetadata.getAllFreshGroups(), groups -> { List<GroupKey> expiredGroupKeys = new ArrayList<>(); - for (Pair<GroupKey, DataFileGroupInternal> pair : groups) { - GroupKey groupKey = pair.first; - DataFileGroupInternal dataFileGroup = pair.second; + for (GroupKeyAndGroup pair : groups) { + GroupKey groupKey = pair.groupKey(); + DataFileGroupInternal dataFileGroup = pair.dataFileGroup(); Long groupExpirationDateMillis = FileGroupUtil.getExpirationDateMillis(dataFileGroup); LogUtil.d( "%s: Checking group %s with expiration date %s", TAG, dataFileGroup.getGroupName(), groupExpirationDateMillis); if (FileGroupUtil.isExpired(groupExpirationDateMillis, timeSource)) { eventLogger.logEventSampled( - 0, + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, dataFileGroup.getGroupName(), dataFileGroup.getFileGroupVersionNumber(), dataFileGroup.getBuildId(), @@ -147,7 +147,7 @@ public class ExpirationHandler { fileGroupsMetadata.removeAllGroupsWithKeys(expiredGroupKeys), removeSuccess -> { if (!removeSuccess) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); LogUtil.e("%s: Failed to remove expired groups!", TAG); } return null; @@ -173,7 +173,7 @@ public class ExpirationHandler { // Remove the group from this list if its expired. if (FileGroupUtil.isExpired(actualExpirationDateMillis, timeSource)) { eventLogger.logEventSampled( - 0, + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, staleGroup.getGroupName(), staleGroup.getFileGroupVersionNumber(), staleGroup.getBuildId(), @@ -197,7 +197,7 @@ public class ExpirationHandler { fileGroupsMetadata.writeStaleGroups(nonExpiredStaleGroups), writeSuccess -> { if (!writeSuccess) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); LogUtil.e("%s: Failed to write back stale groups!", TAG); } return immediateVoidFuture(); @@ -239,7 +239,8 @@ public class ExpirationHandler { if (success) { removedMetadataCount.getAndIncrement(); } else { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); LogUtil.e( "%s: Unsubscribe from file %s failed!", TAG, newFileKey); @@ -325,8 +326,8 @@ public class ExpirationHandler { allGroupsByKey -> { Set<NewFileKey> fileKeysReferencedByAnyGroup = new HashSet<>(); List<DataFileGroupInternal> dataFileGroups = new ArrayList<>(); - for (Pair<GroupKey, DataFileGroupInternal> dataFileGroupPair : allGroupsByKey) { - dataFileGroups.add(dataFileGroupPair.second); + for (GroupKeyAndGroup dataFileGroupPair : allGroupsByKey) { + dataFileGroups.add(dataFileGroupPair.dataFileGroup()); } return PropagatedFutures.transform( fileGroupsMetadata.getAllStaleGroups(), @@ -364,8 +365,8 @@ public class ExpirationHandler { return PropagatedFutures.transform( fileGroupsMetadata.getAllFreshGroups(), groupKeyAndGroupList -> { - for (Pair<GroupKey, DataFileGroupInternal> groupKeyAndGroup : groupKeyAndGroupList) { - DataFileGroupInternal freshGroup = groupKeyAndGroup.second; + for (GroupKeyAndGroup groupKeyAndGroup : groupKeyAndGroupList) { + DataFileGroupInternal freshGroup = groupKeyAndGroup.dataFileGroup(); // Skip any groups that don't support isolated structures if (!FileGroupUtil.isIsolatedStructureAllowed(freshGroup)) { continue; @@ -390,9 +391,9 @@ public class ExpirationHandler { try { fileStorage.deleteFile(sharedFile); releasedFiles += 1; - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); } catch (IOException e) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); LogUtil.e(e, "%s: Failed to release unaccounted file!", TAG); } } @@ -422,13 +423,13 @@ public class ExpirationHandler { } } catch (IOException e) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); LogUtil.e(e, "%s: Failed to delete unaccounted file!", TAG); } } } catch (IOException e) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); LogUtil.e(e, "%s: Failed to delete unaccounted file!", TAG); } return unaccountedFileCount; diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/FileGroupManager.java b/java/com/google/android/libraries/mobiledatadownload/internal/FileGroupManager.java index 1ec6eca..7cf3eb6 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/FileGroupManager.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/FileGroupManager.java @@ -17,6 +17,7 @@ package com.google.android.libraries.mobiledatadownload.internal; import static com.google.android.libraries.mobiledatadownload.tracing.TracePropagation.propagateAsyncFunction; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.util.concurrent.Futures.getDone; import static com.google.common.util.concurrent.Futures.immediateFailedFuture; import static com.google.common.util.concurrent.Futures.immediateFuture; import static com.google.common.util.concurrent.Futures.immediateVoidFuture; @@ -30,7 +31,6 @@ import android.net.Uri; import android.os.Build.VERSION; import android.os.Build.VERSION_CODES; import android.text.TextUtils; -import android.util.Pair; import androidx.annotation.RequiresApi; import com.google.android.libraries.mobiledatadownload.AccountSource; import com.google.android.libraries.mobiledatadownload.AggregateException; @@ -44,8 +44,11 @@ import com.google.android.libraries.mobiledatadownload.account.AccountUtil; import com.google.android.libraries.mobiledatadownload.annotations.InstanceId; import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; import com.google.android.libraries.mobiledatadownload.internal.annotations.SequentialControlExecutor; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupPair; import com.google.android.libraries.mobiledatadownload.internal.experimentation.DownloadStageManager; import com.google.android.libraries.mobiledatadownload.internal.logging.DownloadStateLogger; +import com.google.android.libraries.mobiledatadownload.internal.logging.DownloadStateLogger.Operation; import com.google.android.libraries.mobiledatadownload.internal.logging.EventLogger; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.android.libraries.mobiledatadownload.internal.util.AndroidSharingUtil; @@ -53,9 +56,9 @@ import com.google.android.libraries.mobiledatadownload.internal.util.AndroidShar import com.google.android.libraries.mobiledatadownload.internal.util.DirectoryUtil; import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupUtil; import com.google.android.libraries.mobiledatadownload.internal.util.SymlinkUtil; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedExecutionSequencer; import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentFuture; import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; -import com.google.auto.value.AutoValue; import com.google.common.base.Function; import com.google.common.base.Optional; import com.google.common.base.Preconditions; @@ -63,6 +66,7 @@ import com.google.common.collect.ComparisonChain; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.util.concurrent.AsyncFunction; import com.google.common.util.concurrent.FutureCallback; @@ -82,6 +86,9 @@ import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; import com.google.mobiledatadownload.internal.MetadataProto.GroupKeyProperties; import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey; import com.google.mobiledatadownload.internal.MetadataProto.SharedFile; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; +import com.google.mobiledatadownload.LogEnumsProto.MddDownloadResult; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; import com.google.protobuf.Any; import java.io.IOException; import java.io.PrintWriter; @@ -91,6 +98,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicReference; @@ -117,6 +125,9 @@ public class FileGroupManager { /** The download of at least one file failed. */ FAILED, + + /** The status of the group is unknown. */ + UNKNOWN, } private static final String TAG = "FileGroupManager"; @@ -134,6 +145,10 @@ public class FileGroupManager { private final DownloadStageManager downloadStageManager; private final Flags flags; + // Create an internal ExecutionSequencer to ensure that certain operations remain synced. + private final PropagatedExecutionSequencer futureSerializer = + PropagatedExecutionSequencer.create(); + @Inject public FileGroupManager( @ApplicationContext Context context, @@ -176,18 +191,22 @@ public class FileGroupManager { @SuppressWarnings("nullness") public ListenableFuture<Boolean> addGroupForDownload( GroupKey groupKey, DataFileGroupInternal receivedGroup) - throws ExpiredFileGroupException, IOException, UninstalledAppException, + throws ExpiredFileGroupException, + IOException, + UninstalledAppException, ActivationRequiredForGroupException { if (FileGroupUtil.isActiveGroupExpired(receivedGroup, timeSource)) { LogUtil.e("%s: Trying to add expired group %s.", TAG, groupKey.getGroupName()); - logEventWithDataFileGroup(0, eventLogger, receivedGroup); + logEventWithDataFileGroup( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, eventLogger, receivedGroup); throw new ExpiredFileGroupException(); } if (!isAppInstalled(groupKey.getOwnerPackage())) { LogUtil.e( "%s: Trying to add group %s for uninstalled app %s.", TAG, groupKey.getGroupName(), groupKey.getOwnerPackage()); - logEventWithDataFileGroup(0, eventLogger, receivedGroup); + logEventWithDataFileGroup( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, eventLogger, receivedGroup); throw new UninstalledAppException(); } @@ -210,7 +229,8 @@ public class FileGroupManager { "%s: Trying to add group %s that requires activation %s.", TAG, groupKey.getGroupName(), groupKey.getOwnerPackage()); - logEventWithDataFileGroup(0, eventLogger, receivedGroup); + logEventWithDataFileGroup( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, eventLogger, receivedGroup); throw new ActivationRequiredForGroupException(); } @@ -222,19 +242,34 @@ public class FileGroupManager { .transformAsync( voidArg -> isAddedGroupDuplicate(groupKey, receivedGroup), sequentialControlExecutor) .transformAsync( - isDuplicate -> { - if (isDuplicate) { + newConfigReason -> { + if (!newConfigReason.isPresent()) { + // Absent reason means the config is not new LogUtil.d( "%s: Received duplicate config for group: %s", TAG, groupKey.getGroupName()); return immediateFuture(false); } + + // If supported, set the isolated root before writing to metadata + DataFileGroupInternal receivedGroupWithIsolatedRoot = + FileGroupUtil.maybeSetIsolatedRoot(receivedGroup, groupKey); + return transformSequentialAsync( - maybeSetGroupNewFilesReceivedTimestamp(groupKey, receivedGroup), + maybeSetGroupNewFilesReceivedTimestamp(groupKey, receivedGroupWithIsolatedRoot), receivedGroupCopy -> { LogUtil.d( "%s: Received new config for group: %s", TAG, groupKey.getGroupName()); - logEventWithDataFileGroup(0, eventLogger, receivedGroupCopy); + eventLogger.logNewConfigReceived( + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(receivedGroupCopy.getGroupName()) + .setOwnerPackage(receivedGroupCopy.getOwnerPackage()) + .setFileGroupVersionNumber( + receivedGroupCopy.getFileGroupVersionNumber()) + .setBuildId(receivedGroupCopy.getBuildId()) + .setVariantId(receivedGroupCopy.getVariantId()) + .build(), + null); return transformSequentialAsync( subscribeGroup(receivedGroupCopy), @@ -276,7 +311,7 @@ public class FileGroupManager { .transformAsync( writeSuccess -> { if (!writeSuccess) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return immediateFailedFuture( new IOException("Failed to commit new group metadata to disk.")); } @@ -335,7 +370,8 @@ public class FileGroupManager { "%s: Failed to remove pending version for group: '%s';" + " account: '%s'", TAG, groupKey.getGroupName(), groupKey.getAccount()); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return immediateFailedFuture( new IOException( "Failed to remove pending group: " @@ -364,7 +400,8 @@ public class FileGroupManager { "%s: Failed to remove the downloaded version for group:" + " '%s'; account: '%s'", TAG, groupKey.getGroupName(), groupKey.getAccount()); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return immediateFailedFuture( new IOException( "Failed to remove downloaded group: " @@ -379,7 +416,8 @@ public class FileGroupManager { "%s: Failed to add to stale for group: '%s';" + " account: '%s'", TAG, groupKey.getGroupName(), groupKey.getAccount()); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return immediateFailedFuture( new IOException( "Failed to add downloaded group to stale: " @@ -512,7 +550,8 @@ public class FileGroupManager { "%s: Failed to remove %d pending versions of %d requested" + " groups", TAG, pendingGroupsToRemove.size(), groupKeys.size()); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return immediateFailedFuture( new IOException( "Failed to remove pending group keys, count = " @@ -565,7 +604,8 @@ public class FileGroupManager { "%s: Failed to remove %d downloaded versions of %d requested" + " groups", TAG, downloadedGroupsToRemove.size(), groupKeys.size()); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return immediateFailedFuture( new IOException( "Failed to remove downloaded groups, count = " @@ -598,7 +638,7 @@ public class FileGroupManager { LogUtil.e( "%s: Failed to add to stale for group: '%s';", TAG, staleGroup.getGroupName()); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return immediateFailedFuture( new IOException( "Failed to add downloaded group to stale: " @@ -668,15 +708,7 @@ public class FileGroupManager { public ListenableFuture<@NullableType DataFileGroupInternal> getFileGroup( GroupKey groupKey, boolean downloaded) { GroupKey downloadedKey = groupKey.toBuilder().setDownloaded(downloaded).build(); - return transformSequentialAsync( - fileGroupsMetadata.read(downloadedKey), - dataFileGroup -> - transformSequentialAsync( - // TODO(b/194688687): consider moving this verification to the - // MobileDataDownloadManager level since that is where verification happens for - // getDataFileUri. - maybeVerifyIsolatedStructure(dataFileGroup, downloaded), - result -> immediateFuture(result ? dataFileGroup : null))); + return fileGroupsMetadata.read(downloadedKey); } /** @@ -687,25 +719,24 @@ public class FileGroupManager { * pending/downloded states of a file group, so the downloaded status in the given groupKey is not * considered by this method. * - * <p>If a group is found, state of the file group (downloaded/pending) and file group will be - * returned in a Pair. If a group is not found, null will be returned. The boolean returned will - * be true if the group is downloaded and false if the group is pending. + * <p>If a group is found, a {@link GroupKeyAndGroup} will be returned. If a group is not found, + * null will be returned. The boolean returned will be true if the group is downloaded and false + * if the group is pending. * * @param groupKey The key for the data to be returned. This is should include group name, owner * package and user account * @param buildId The expected buildId of the file group * @param variantId The expected variantId of the file group * @param customPropertyOptional The expected customProperty, if necessary - * @return A ListenableFuture that resolves, if the requested group is found, with a Pair - * containing Boolean value of whether or not the Group is downloaded and the Group itself, or - * null otherwise. + * @return A ListenableFuture that resolves, if the requested group is found, to a {@link + * GroupKeyAndGroup}, or null if no group is found. */ - private ListenableFuture<@NullableType Pair<Boolean, DataFileGroupInternal>> getGroupPairById( + private ListenableFuture<@NullableType GroupKeyAndGroup> getGroupPairById( GroupKey groupKey, long buildId, String variantId, Optional<Any> customPropertyOptional) { return transformSequential( fileGroupsMetadata.getAllFreshGroups(), freshGroupPairList -> { - for (Pair<GroupKey, DataFileGroupInternal> freshGroupPair : freshGroupPairList) { + for (GroupKeyAndGroup freshGroupPair : freshGroupPairList) { if (!verifyGroupPairMatchesIdentifiers( freshGroupPair, groupKey.getAccount(), @@ -717,19 +748,19 @@ public class FileGroupManager { } // Group matches ID, but ensure that it also matches requested group name - if (!groupKey.getGroupName().equals(freshGroupPair.first.getGroupName())) { + if (!groupKey.getGroupName().equals(freshGroupPair.groupKey().getGroupName())) { LogUtil.e( "%s: getGroupPairById: Group %s matches the given buildId = %d and variantId =" + " %s, but does not match the given group name %s", TAG, - freshGroupPair.first.getGroupName(), + freshGroupPair.groupKey().getGroupName(), buildId, variantId, groupKey.getGroupName()); continue; } - return Pair.create(freshGroupPair.first.getDownloaded(), freshGroupPair.second); + return freshGroupPair; } // No compatible group found, return null; @@ -790,7 +821,7 @@ public class FileGroupManager { fileGroupsMetadata.remove(groupKey), removeSuccess -> { if (!removeSuccess) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); } return immediateVoidFuture(); }); @@ -842,11 +873,11 @@ public class FileGroupManager { DownloadStateLogger downloadStateLogger = DownloadStateLogger.forImport(eventLogger); // Get group that should be updated for import, or return group not found failure - ListenableFuture<Pair<Boolean, DataFileGroupInternal>> groupPairToUpdateFuture = + ListenableFuture<GroupKeyAndGroup> groupKeyAndGroupToUpdateFuture = transformSequentialAsync( getGroupPairById(groupKey, buildId, variantId, customPropertyOptional), - foundGroupPair -> { - if (foundGroupPair == null) { + foundGroupKeyAndGroup -> { + if (foundGroupKeyAndGroup == null) { // Group with identifiers could not be found, return failure. LogUtil.e( "%s: importFiles for group name: %s, buildId: %d, variantId: %s, but no group" @@ -863,16 +894,17 @@ public class FileGroupManager { } // wrap in checkNotNull to ensure type safety. - return immediateFuture(checkNotNull(foundGroupPair)); + return immediateFuture(checkNotNull(foundGroupKeyAndGroup)); }); - return PropagatedFluentFuture.from(groupPairToUpdateFuture) + return PropagatedFluentFuture.from(groupKeyAndGroupToUpdateFuture) .transformAsync( - groupPairToUpdate -> { + groupKeyAndGroupToUpdate -> { // Perform an in-memory merge of updatedDataFileList into the group, so we get the // correct list of files to import. DataFileGroupInternal mergedFileGroup = - mergeFilesIntoFileGroup(updatedDataFileList, groupPairToUpdate.second); + mergeFilesIntoFileGroup( + updatedDataFileList, groupKeyAndGroupToUpdate.dataFileGroup()); // Log the start of the import now that we have the group. downloadStateLogger.logStarted(mergedFileGroup); @@ -898,7 +930,8 @@ public class FileGroupManager { sequentialControlExecutor) .transformAsync( mergedFileGroup -> { - boolean groupIsDownloaded = Futures.getDone(groupPairToUpdateFuture).first; + boolean groupIsDownloaded = + Futures.getDone(groupKeyAndGroupToUpdateFuture).groupKey().getDownloaded(); // If we are updating a pending group and the import is successful, the pending // version should be removed from metadata. @@ -913,12 +946,15 @@ public class FileGroupManager { PropagatedFutures.whenAllComplete(allImportFutures) .callAsync( () -> - verifyGroupDownloaded( - groupKey, - mergedFileGroup, - removePendingVersion, - customFileGroupValidator, - downloadStateLogger), + futureSerializer.submitAsync( + () -> + verifyGroupDownloaded( + groupKey, + mergedFileGroup, + removePendingVersion, + customFileGroupValidator, + downloadStateLogger), + sequentialControlExecutor), sequentialControlExecutor); return transformSequentialAsync( combinedImportFuture, @@ -932,7 +968,16 @@ public class FileGroupManager { // We log other results in verifyGroupDownloaded, so only check for // downloaded here. if (groupDownloadStatus == GroupDownloadStatus.DOWNLOADED) { - eventLogger.logMddDownloadResult(0, null); + eventLogger.logMddDownloadResult( + MddDownloadResult.Code.SUCCESS, + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(groupKey.getGroupName()) + .setOwnerPackage(groupKey.getOwnerPackage()) + .setFileGroupVersionNumber( + mergedFileGroup.getFileGroupVersionNumber()) + .setBuildId(mergedFileGroup.getBuildId()) + .setVariantId(mergedFileGroup.getVariantId()) + .build()); // group downloaded, so it will be written in verifyGroupDownloaded, return // early. return immediateVoidFuture(); @@ -948,7 +993,7 @@ public class FileGroupManager { mergedFileGroup), writeSuccess -> { if (!writeSuccess) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return immediateFailedFuture( DownloadException.builder() .setMessage( @@ -1016,13 +1061,13 @@ public class FileGroupManager { * </ul> */ private static boolean verifyGroupPairMatchesIdentifiers( - Pair<GroupKey, DataFileGroupInternal> groupPair, + GroupKeyAndGroup groupPair, String serializedAccount, long buildId, String variantId, Optional<Any> customPropertyOptional) { - DataFileGroupInternal fileGroup = groupPair.second; - if (!groupPair.first.getAccount().equals(serializedAccount)) { + DataFileGroupInternal fileGroup = groupPair.dataFileGroup(); + if (!groupPair.groupKey().getAccount().equals(serializedAccount)) { LogUtil.v( "%s: verifyGroupPairMatchesIdentifiers failed for group %s due to mismatched account", TAG, fileGroup.getGroupName()); @@ -1211,17 +1256,42 @@ public class FileGroupManager { return PropagatedFutures.whenAllComplete(allFileFutures) .callAsync( () -> - transformSequentialAsync( - verifyPendingGroupDownloaded( - groupKey, - updatedPendingGroup, - customFileGroupValidator), - groupDownloadStatus -> - finalizeDownloadFileFutures( - allFileFutures, - groupDownloadStatus, - updatedPendingGroup, - groupKey)), + futureSerializer.submitAsync( + () -> + transformSequentialAsync( + getGroupPair(groupKey), + groupPair -> { + @NullableType + DataFileGroupInternal groupToVerify = + groupPair.pendingGroup() != null + ? groupPair.pendingGroup() + : groupPair.downloadedGroup(); + if (groupToVerify != null) { + return transformSequentialAsync( + verifyGroupDownloaded( + groupKey, + groupToVerify, + /* removePendingVersion= */ true, + customFileGroupValidator, + DownloadStateLogger.forDownload( + eventLogger)), + groupDownloadStatus -> + finalizeDownloadFileFutures( + allFileFutures, + groupDownloadStatus, + groupToVerify, + groupKey)); + } else { + // No group to verify, which should be + // impossible -- force a failure state so we can + // track any download file failures. + handleDownloadFileFutureFailures( + allFileFutures, groupKey); + return immediateFailedFuture( + new AssertionError("impossible error")); + } + }), + sequentialControlExecutor), sequentialControlExecutor); }, sequentialControlExecutor); @@ -1281,6 +1351,24 @@ public class FileGroupManager { sequentialControlExecutor); } + private ListenableFuture<GroupPair> getGroupPair(GroupKey groupKey) { + return PropagatedFutures.submitAsync( + () -> { + ListenableFuture<@NullableType DataFileGroupInternal> pendingGroupFuture = + getFileGroup(groupKey, /* downloaded= */ false); + ListenableFuture<@NullableType DataFileGroupInternal> downloadedGroupFuture = + getFileGroup(groupKey, /* downloaded= */ true); + return PropagatedFutures.whenAllSucceed(pendingGroupFuture, downloadedGroupFuture) + .callAsync( + () -> + immediateFuture( + GroupPair.create( + getDone(pendingGroupFuture), getDone(downloadedGroupFuture))), + sequentialControlExecutor); + }, + sequentialControlExecutor); + } + private List<ListenableFuture<Void>> startDownloadFutures( @Nullable DownloadConditions downloadConditions, DataFileGroupInternal pendingGroup, @@ -1364,25 +1452,40 @@ public class FileGroupManager { // TODO(b/136112848): When all fileFutures succeed, we don't need to verify them again. However // we still need logic to remove pending and update stale group. if (groupDownloadStatus != GroupDownloadStatus.DOWNLOADED) { - LogUtil.e( - "%s downloadFileGroup %s %s can't finish!", - TAG, groupKey.getGroupName(), groupKey.getOwnerPackage()); - - AggregateException.throwIfFailed( - allFileFutures, "Failed to download file group %s", groupKey.getGroupName()); - - // TODO(b/118137672): Investigate on the unknown error that we've missed. There is a download - // failure that we don't recognize. - LogUtil.e("%s: An unknown error has occurred during" + " download", TAG); - throw DownloadException.builder() - .setDownloadResultCode(DownloadResultCode.UNKNOWN_ERROR) - .build(); + handleDownloadFileFutureFailures(allFileFutures, groupKey); } - eventLogger.logMddDownloadResult(0, null); + eventLogger.logMddDownloadResult( + MddDownloadResult.Code.SUCCESS, + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(groupKey.getGroupName()) + .setOwnerPackage(groupKey.getOwnerPackage()) + .setFileGroupVersionNumber(pendingGroup.getFileGroupVersionNumber()) + .setBuildId(pendingGroup.getBuildId()) + .setVariantId(pendingGroup.getVariantId()) + .build()); return immediateFuture(pendingGroup); } + // Requires that all futures in allFileFutures are completed. + private void handleDownloadFileFutureFailures( + List<ListenableFuture<Void>> allFileFutures, GroupKey groupKey) + throws DownloadException, AggregateException { + LogUtil.e( + "%s downloadFileGroup %s %s can't finish!", + TAG, groupKey.getGroupName(), groupKey.getOwnerPackage()); + + AggregateException.throwIfFailed( + allFileFutures, "Failed to download file group %s", groupKey.getGroupName()); + + // TODO(b/118137672): Investigate on the unknown error that we've missed. There is a download + // failure that we don't recognize. + LogUtil.e("%s: An unknown error has occurred during" + " download", TAG); + throw DownloadException.builder() + .setDownloadResultCode(DownloadResultCode.UNKNOWN_ERROR) + .build(); + } + /** * If the file is available in the shared blob storage, it acquires the lease and updates the * shared file metadata. The {@code FileStatus} will be set to DOWNLOAD_COMPLETE so that the file @@ -1475,7 +1578,7 @@ public class FileGroupManager { fileGroup, dataFile, fileStorage, - /* afterDownload = */ false); + /* afterDownload= */ false); return transformSequentialAsync( maybeUpdateLeaseAndSharedMetadata( fileGroup, @@ -1589,7 +1692,6 @@ public class FileGroupManager { 0), res -> { if (res) { - deleteLocalCopy(downloadFileOnDeviceUri, fileGroup, dataFile); return immediateVoidFuture(); } return updateMaxExpirationDateSecs( @@ -1610,7 +1712,7 @@ public class FileGroupManager { fileGroup, dataFile, fileStorage, - /* afterDownload = */ true); + /* afterDownload= */ true); return transformSequentialAsync( maybeUpdateLeaseAndSharedMetadata( fileGroup, @@ -1622,7 +1724,6 @@ public class FileGroupManager { 0), res -> { if (res) { - deleteLocalCopy(downloadFileOnDeviceUri, fileGroup, dataFile); return immediateVoidFuture(); } return updateMaxExpirationDateSecs( @@ -1740,7 +1841,7 @@ public class FileGroupManager { dataFile.getChecksum(), silentFeedback, instanceId, - /* androidShared = */ false); + /* androidShared= */ false); if (downloadFileOnDeviceUri == null) { LogUtil.e("%s: Failed to get file uri!", TAG); throw new AndroidSharingException(0, "Failed to get local file uri"); @@ -1748,19 +1849,6 @@ public class FileGroupManager { return downloadFileOnDeviceUri; } - private void deleteLocalCopy( - Uri downloadFileOnDeviceUri, DataFileGroupInternal fileGroup, DataFile dataFile) { - try { - fileStorage.deleteFile(downloadFileOnDeviceUri); - } catch (IOException e) { - LogUtil.e( - "%s: Failed to delete the local copy after android-sharing the file" - + " %s, file group %s", - TAG, dataFile.getFileId(), fileGroup.getGroupName()); - logMddAndroidSharingLog(eventLogger, fileGroup, dataFile, 0); - } - } - /** * Download and Verify all files present in any pending groups. * @@ -1842,30 +1930,8 @@ public class FileGroupManager { } /** - * Verifies that the given pending group was downloaded, and updates the metadata if the download - * has completed. - * - * @param groupKey The key of the group to verify for download. - * @param pendingGroup The group to verify for download. - * @return A future that resolves to true if the given group was verify for download, false - * otherwise. - */ - // TODO(b/124072754): Change to package private once all code is refactored. - public ListenableFuture<GroupDownloadStatus> verifyPendingGroupDownloaded( - GroupKey groupKey, - DataFileGroupInternal pendingGroup, - AsyncFunction<DataFileGroupInternal, Boolean> customFileGroupValidator) { - return verifyGroupDownloaded( - groupKey, - pendingGroup, - /* removePendingVersion = */ true, - customFileGroupValidator, - /* downloadStateLogger = */ DownloadStateLogger.forDownload(eventLogger)); - } - - /** - * Verifies that the given pending group was downloaded, and updates the metadata if the download - * has completed. + * Verifies that the given group was downloaded, and updates the metadata if the download has + * completed. * * @param groupKey The key of the group to verify for download. * @param fileGroup The group to verify for download. @@ -1874,7 +1940,7 @@ public class FileGroupManager { * @return A future that resolves to true if the given group was verify for download, false * otherwise. */ - private ListenableFuture<GroupDownloadStatus> verifyGroupDownloaded( + ListenableFuture<GroupDownloadStatus> verifyGroupDownloaded( GroupKey groupKey, DataFileGroupInternal fileGroup, boolean removePendingVersion, @@ -1887,6 +1953,11 @@ public class FileGroupManager { GroupKey downloadedGroupKey = groupKey.toBuilder().setDownloaded(true).build(); GroupKey pendingGroupKey = groupKey.toBuilder().setDownloaded(false).build(); + // It's possible that we are calling verifyGroupDownloaded concurrently, which would lead to + // multiple DOWNLOAD_COMPLETE logs. To prevent this, we check to see if we've already logged the + // timestamp so we can skip logging later. + boolean completeAlreadyLogged = + fileGroup.getBookkeeping().hasGroupDownloadedTimestampInMillis(); DataFileGroupInternal downloadedFileGroupWithTimestamp = FileGroupUtil.setDownloadedTimestampInMillis(fileGroup, timeSource.currentTimeMillis()); @@ -1917,6 +1988,8 @@ public class FileGroupManager { // supported if (FileGroupUtil.isIsolatedStructureAllowed(fileGroup) && VERSION.SDK_INT >= VERSION_CODES.LOLLIPOP) { + // TODO(b/225409326): Prevent race condition where recreation of isolated + // paths happens at the same time as group access. return createIsolatedFilePaths(fileGroup); } return immediateVoidFuture(); @@ -1939,7 +2012,12 @@ public class FileGroupManager { .transformAsync(this::addGroupAsStaleIfPresent, sequentialControlExecutor) .transform( voidArg -> { - downloadStateLogger.logComplete(downloadedFileGroupWithTimestamp); + // Only log complete if we are performing an import operation OR we haven't + // already logged a download complete event. + if (!completeAlreadyLogged + || downloadStateLogger.getOperation() == Operation.IMPORT) { + downloadStateLogger.logComplete(downloadedFileGroupWithTimestamp); + } return GroupDownloadStatus.DOWNLOADED; }, sequentialControlExecutor); @@ -1966,7 +2044,7 @@ public class FileGroupManager { .transformAsync( writeSuccess -> { if (!writeSuccess) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return immediateFailedFuture( new IOException( "Failed to write updated group: " + downloadedGroupKey.getGroupName())); @@ -1985,7 +2063,7 @@ public class FileGroupManager { fileGroupsMetadata.remove(pendingGroupKey), removeSuccess -> { if (!removeSuccess) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); } return toReturn; }); @@ -2019,7 +2097,7 @@ public class FileGroupManager { "%s: Failed to remove pending version for group: '%s';" + " account: '%s'", TAG, pendingGroupKey.getGroupName(), pendingGroupKey.getAccount()); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return immediateFailedFuture( new IOException( "Failed to remove pending group: " + pendingGroupKey.getGroupName())); @@ -2050,7 +2128,7 @@ public class FileGroupManager { // unaccounted for, and the files will get deleted // in the next daily maintenance, hence not // enforcing its stale lifetime. - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); } return immediateVoidFuture(); }); @@ -2087,28 +2165,31 @@ public class FileGroupManager { .setCause(e) .build()); } - List<ListenableFuture<Void>> createSymlinkFutures = - new ArrayList<>(dataFileGroup.getFileCount()); - for (DataFile dataFile : dataFileGroup.getFileList()) { - if (dataFile.getAndroidSharingType() == AndroidSharingType.ANDROID_BLOB_WHEN_AVAILABLE) { - createSymlinkFutures.add( - immediateFailedFuture( - new UnsupportedOperationException( - "Preserve File Paths is invalid with Android Blob Sharing"))); - // break out of loop since we've already hit a failure. - break; - } + List<DataFile> dataFiles = dataFileGroup.getFileList(); - // Get the original path - ListenableFuture<Void> createSymlinkFuture = - transformSequentialAsync( - getOnDeviceUri(dataFile, dataFileGroup), - (Uri originalUri) -> { - Uri symlinkUri = - FileGroupUtil.getIsolatedFileUri(context, instanceId, dataFile, dataFileGroup); + if (Iterables.tryFind( + dataFiles, + dataFile -> + dataFile.getAndroidSharingType() == AndroidSharingType.ANDROID_BLOB_WHEN_AVAILABLE) + .isPresent()) { + // Creating isolated structure is not supported when android sharing is enabled in the group; + // return immediately. + return immediateFailedFuture( + new UnsupportedOperationException( + "Preserve File Paths is invalid with Android Blob Sharing")); + } + ImmutableMap<DataFile, Uri> isolatedFileUriMap = getIsolatedFileUris(dataFileGroup); + ListenableFuture<Void> createIsolatedStructureFuture = + PropagatedFutures.transformAsync( + getOnDeviceUris(dataFileGroup), + onDeviceUriMap -> { + for (DataFile dataFile : dataFiles) { try { + Uri symlinkUri = checkNotNull(isolatedFileUriMap.get(dataFile)); + Uri originalUri = checkNotNull(onDeviceUriMap.get(dataFile)); + // Check/create parent dir of symlink. Uri symlinkParentDir = Uri.parse( @@ -2118,8 +2199,8 @@ public class FileGroupManager { if (!fileStorage.exists(symlinkParentDir)) { fileStorage.createDirectory(symlinkParentDir); } - SymlinkUtil.createSymlink(context, symlinkUri, checkNotNull(originalUri)); - } catch (IOException e) { + SymlinkUtil.createSymlink(context, symlinkUri, originalUri); + } catch (NullPointerException | IOException e) { return immediateFailedFuture( DownloadException.builder() .setDownloadResultCode( @@ -2128,15 +2209,13 @@ public class FileGroupManager { .setCause(e) .build()); } - return immediateVoidFuture(); - }); - createSymlinkFutures.add(createSymlinkFuture); - } - ListenableFuture<Void> combinedFuture = - Futures.whenAllSucceed(createSymlinkFutures).call(() -> null, sequentialControlExecutor); + } + return immediateVoidFuture(); + }, + sequentialControlExecutor); PropagatedFutures.addCallback( - combinedFuture, + createIsolatedStructureFuture, new FutureCallback<Void>() { @Override public void onSuccess(Void unused) {} @@ -2155,29 +2234,7 @@ public class FileGroupManager { }, sequentialControlExecutor); - return combinedFuture; - } - - /** - * Gets the Isolated File Uri and verifies that it exists and points to the given uri. - * - * <p>Throws IOException when verifying the symlink fails. - */ - @RequiresApi(VERSION_CODES.LOLLIPOP) - Uri getAndVerifyIsolatedFileUri( - Uri originalFileUri, DataFile dataFile, DataFileGroupInternal dataFileGroup) - throws IOException { - Uri isolatedFileUri = - FileGroupUtil.getIsolatedFileUri(context, instanceId, dataFile, dataFileGroup); - - Uri targetFileUri = SymlinkUtil.readSymlink(context, isolatedFileUri); - - if (!fileStorage.exists(isolatedFileUri) - || !targetFileUri.toString().equals(originalFileUri.toString())) { - throw new IOException("Isolated file uri does not exist or points to an unexpected target"); - } - - return isolatedFileUri; + return createIsolatedStructureFuture; } /** @@ -2201,7 +2258,7 @@ public class FileGroupManager { * * <p>This method is annotated with @TargetApi(21) since symlink structure methods require API * level 21 or later. The FileGroupUtil.isIsolatedStructureAllowed check will ensure this - * condition is met before calling getAndVerifyIsolatedFileUri and createIsolatedFilePaths. + * condition is met before calling verifyIsolatedFileUris and createIsolatedFilePaths. * * @return Future that resolves to true if the isolated structure is verified, or false if the * structure couldn't be verified @@ -2217,36 +2274,24 @@ public class FileGroupManager { return immediateFuture(true); } - List<ListenableFuture<Void>> verifyIsolatedFileFutures = - new ArrayList<>(dataFileGroup.getFileCount()); - for (DataFile dataFile : dataFileGroup.getFileList()) { - verifyIsolatedFileFutures.add( - transformSequentialAsync( - getOnDeviceUri(dataFile, dataFileGroup), - onDeviceUri -> { - if (onDeviceUri != null) { - Uri unused = getAndVerifyIsolatedFileUri(onDeviceUri, dataFile, dataFileGroup); + return PropagatedFluentFuture.from(getOnDeviceUris(dataFileGroup)) + .transform( + onDeviceUriMap -> { + ImmutableMap<DataFile, Uri> verifiedUriMap = + verifyIsolatedFileUris(getIsolatedFileUris(dataFileGroup), onDeviceUriMap); + for (DataFile dataFile : dataFileGroup.getFileList()) { + if (!verifiedUriMap.containsKey(dataFile)) { + // File is missing from map, so verification failed, log this error and return + // false. + LogUtil.w( + "%s: Detected corruption of isolated structure for group %s %s", + TAG, dataFileGroup.getGroupName(), dataFile.getFileId()); + return false; } - return immediateVoidFuture(); - })); - } - - return PropagatedFutures.catching( - Futures.whenAllSucceed(verifyIsolatedFileFutures) - .call(() -> true, sequentialControlExecutor), - IOException.class, - ex -> { - // TODO(b/194688687): Log these events to clearcut along with their file group info so - // we can understand how often this is happening. - LogUtil.w( - ex, - "%s: Detected corruption of isolated structure for group %s", - TAG, - dataFileGroup.getGroupName()); - - return false; - }, - sequentialControlExecutor); + } + return true; + }, + sequentialControlExecutor); } /** @@ -2272,6 +2317,119 @@ public class FileGroupManager { } /** + * Gets the on-device uri of the given list of {@link DataFile}s. + * + * <p>Checks for sideloading support. If the file is sideloaded and sideloading is enabled, the + * sideloaded uri will be returned immediately. If sideloading is not enabled, returns a faliure. + * + * <p>If file is not sideloaded, delegates to {@link SharedFileManager#getOnDeviceUris()}. + * + * <p>NOTE: The returned map will contain entries for all data files with a known uri. If the uri + * is unable to be calculated, it will not be included in the returned list. + */ + ListenableFuture<ImmutableMap<DataFile, Uri>> getOnDeviceUris( + DataFileGroupInternal dataFileGroup) { + ImmutableMap.Builder<DataFile, Uri> onDeviceUriMap = ImmutableMap.builder(); + ImmutableMap.Builder<DataFile, NewFileKey> nonSideloadedKeyMapBuilder = ImmutableMap.builder(); + for (DataFile dataFile : dataFileGroup.getFileList()) { + if (FileGroupUtil.isSideloadedFile(dataFile)) { + // Sideloaded file -- put in map immediately + onDeviceUriMap.put(dataFile, Uri.parse(dataFile.getUrlToDownload())); + } else { + // Non sideloaded file -- mark for further lookup + nonSideloadedKeyMapBuilder.put( + dataFile, + SharedFilesMetadata.createKeyFromDataFile( + dataFile, dataFileGroup.getAllowedReadersEnum())); + } + } + ImmutableMap<DataFile, NewFileKey> nonSideloadedKeyMap = + nonSideloadedKeyMapBuilder.build(); + + return PropagatedFluentFuture.from( + sharedFileManager.getOnDeviceUris(ImmutableSet.copyOf(nonSideloadedKeyMap.values()))) + .transform( + nonSideloadedUriMap -> { + // Extract the <DataFile, Uri> entries from the two non-sideloaded maps. + // DataFile -> NewFileKey -> Uri now becomes DataFile -> Uri + for (Entry<DataFile, NewFileKey> keyMapEntry : nonSideloadedKeyMap.entrySet()) { + NewFileKey newFileKey = keyMapEntry.getValue(); + if (newFileKey != null && nonSideloadedUriMap.containsKey(newFileKey)) { + onDeviceUriMap.put(keyMapEntry.getKey(), nonSideloadedUriMap.get(newFileKey)); + } + } + return onDeviceUriMap.build(); + }, + sequentialControlExecutor); + } + + /** + * Helper method to get a map of isolated file uris. + * + * <p>This method does not check whether or not isolated uris are allowed to be created/used, but + * simply returns all calculated isolated file uris. The caller is responsible for checking if the + * returned uris can/should be used! + */ + ImmutableMap<DataFile, Uri> getIsolatedFileUris(DataFileGroupInternal dataFileGroup) { + ImmutableMap.Builder<DataFile, Uri> isolatedFileUrisBuilder = ImmutableMap.builder(); + Uri isolatedRootUri = + FileGroupUtil.getIsolatedRootDirectory(context, instanceId, dataFileGroup); + for (DataFile dataFile : dataFileGroup.getFileList()) { + isolatedFileUrisBuilder.put( + dataFile, FileGroupUtil.appendIsolatedFileUri(isolatedRootUri, dataFile)); + } + return isolatedFileUrisBuilder.build(); + } + + /** + * Verify the given isolated uris point to the given on-device uris. + * + * <p>The verification steps include 1) ensuring each isolated uri exists; 2) each isolated uri + * points to the corresponding on-device uri. Isolated uris and on-device uris will be matched by + * their {@link DataFile} keys from the input maps. + * + * <p>Each verified isolated uri is included in the return map. If an isolated uri cannot be + * verified, no entry for the corresponding data file will be included in the return map. + * + * <p>If an entry for a DataFile key is missing from either input map, it is also omitted from the + * return map (i.e. this method returns an INNER JOIN of the two input maps) + * + * @return map of isolated uris which have been verified + */ + @RequiresApi(VERSION_CODES.LOLLIPOP) + ImmutableMap<DataFile, Uri> verifyIsolatedFileUris( + ImmutableMap<DataFile, Uri> isolatedFileUris, ImmutableMap<DataFile, Uri> onDeviceUris) { + ImmutableMap.Builder<DataFile, Uri> verifiedUriMapBuilder = ImmutableMap.builder(); + for (Entry<DataFile, Uri> onDeviceEntry : onDeviceUris.entrySet()) { + // Skip null/missing uris + if (onDeviceEntry.getValue() == null + || !isolatedFileUris.containsKey(onDeviceEntry.getKey())) { + continue; + } + + Uri isolatedUri = isolatedFileUris.get(onDeviceEntry.getKey()); + Uri onDeviceUri = onDeviceEntry.getValue(); + + try { + Uri targetFileUri = SymlinkUtil.readSymlink(context, isolatedUri); + if (fileStorage.exists(isolatedUri) + && targetFileUri.toString().equals(onDeviceUri.toString())) { + verifiedUriMapBuilder.put(onDeviceEntry.getKey(), isolatedUri); + } else { + LogUtil.e( + "%s verifyIsolatedFileUris unable to get isolated file uri! %s %s", + TAG, isolatedUri, onDeviceUri); + } + } catch (IOException e) { + LogUtil.e( + "%s verifyIsolatedFileUris unable to get isolated file uri! %s %s", + TAG, isolatedUri, onDeviceUri); + } + } + return verifiedUriMapBuilder.build(); + } + + /** * Get the current status of the file group. Since the status of the group is not stored in the * file group, this method iterates over all files and re-calculates the current status. * @@ -2281,9 +2439,9 @@ public class FileGroupManager { DataFileGroupInternal dataFileGroup) { return getFileGroupDownloadStatusIter( dataFileGroup, - /* downloadFailed = */ false, - /* downloadPending = */ false, - /* index = */ 0, + /* downloadFailed= */ false, + /* downloadPending= */ false, + /* index= */ 0, dataFileGroup.getFileCount()); } @@ -2335,7 +2493,7 @@ public class FileGroupManager { return getFileGroupDownloadStatusIter( dataFileGroup, downloadFailed, - /* downloadPending = */ true, + /* downloadPending= */ true, index + 1, fileCount); } else { @@ -2344,7 +2502,7 @@ public class FileGroupManager { TAG, dataFile.getFileId(), dataFileGroup.getGroupName()); return getFileGroupDownloadStatusIter( dataFileGroup, - /* downloadFailed = */ true, + /* downloadFailed= */ true, downloadPending, index + 1, fileCount); @@ -2376,9 +2534,6 @@ public class FileGroupManager { verifyAllPendingGroupsDownloaded(groupKeyList, customFileGroupValidator))); } - @SuppressWarnings("nullness") - // Suppress nullness warnings because otherwise static analysis would require us to falsely label - // verifyPendingGroupDownloaded with @NullableType private ListenableFuture<Void> verifyAllPendingGroupsDownloaded( List<GroupKey> groupKeyList, AsyncFunction<DataFileGroupInternal, Boolean> customFileGroupValidator) { @@ -2389,13 +2544,18 @@ public class FileGroupManager { } allFileFutures.add( transformSequentialAsync( - fileGroupsMetadata.read(groupKey), + getFileGroup(groupKey, /* downloaded= */ false), pendingGroup -> { + // If no pending group exists for this group key, skip the verification. if (pendingGroup == null) { - return immediateFuture(null); + return immediateFuture(GroupDownloadStatus.PENDING); } - return verifyPendingGroupDownloaded( - groupKey, pendingGroup, customFileGroupValidator); + return verifyGroupDownloaded( + groupKey, + pendingGroup, + /* removePendingVersion= */ true, + customFileGroupValidator, + DownloadStateLogger.forDownload(eventLogger)); })); } return PropagatedFutures.whenAllComplete(allFileFutures) @@ -2420,12 +2580,13 @@ public class FileGroupManager { LogUtil.d( "%s: Deleting file group %s for uninstalled app %s", TAG, key.getGroupName(), key.getOwnerPackage()); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return transformSequentialAsync( fileGroupsMetadata.remove(key), removeSuccess -> { if (!removeSuccess) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); } return immediateVoidFuture(); }); @@ -2474,14 +2635,16 @@ public class FileGroupManager { LogUtil.d( "%s: Deleting file group %s for removed account %s", TAG, key.getGroupName(), key.getOwnerPackage()); - logEventWithDataFileGroup(0, eventLogger, group); + logEventWithDataFileGroup( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, eventLogger, group); // Remove the group from fresh file groups if the account is removed. return transformSequentialAsync( fileGroupsMetadata.remove(key), removeSuccess -> { if (!removeSuccess) { - logEventWithDataFileGroup(0, eventLogger, group); + logEventWithDataFileGroup( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, eventLogger, group); } return immediateVoidFuture(); }); @@ -2523,7 +2686,7 @@ public class FileGroupManager { fileGroupsMetadata.write(pendingGroupKey, pendingGroup), writeSuccess -> { if (!writeSuccess) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return immediateFailedFuture(new IOException("Unable to update file group metadata")); } @@ -2543,8 +2706,8 @@ public class FileGroupManager { return transformSequential( fileGroupsMetadata.getAllFreshGroups(), pairs -> { - for (Pair<GroupKey, DataFileGroupInternal> pair : pairs) { - DataFileGroupInternal fileGroup = pair.second; + for (GroupKeyAndGroup pair : pairs) { + DataFileGroupInternal fileGroup = pair.dataFileGroup(); for (DataFile dataFile : fileGroup.getFileList()) { NewFileKey newFileKey = SharedFilesMetadata.createKeyFromDataFile( @@ -2557,20 +2720,33 @@ public class FileGroupManager { } /** Logs download failure remotely via {@code eventLogger}. */ + // incompatible argument for parameter code of logMddDownloadResult. + @SuppressWarnings("nullness:argument.type.incompatible") private ListenableFuture<Void> logDownloadFailure( GroupKey groupKey, DownloadException downloadException, long buildId, String variantId) { - Void groupDetails = null; + DataDownloadFileGroupStats.Builder groupDetails = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(groupKey.getGroupName()) + .setOwnerPackage(groupKey.getOwnerPackage()) + .setBuildId(buildId) + .setVariantId(variantId); return transformSequentialAsync( fileGroupsMetadata.read(groupKey.toBuilder().setDownloaded(false).build()), dataFileGroup -> { - eventLogger.logMddDownloadResult(0, groupDetails); + if (dataFileGroup != null) { + groupDetails.setFileGroupVersionNumber(dataFileGroup.getFileGroupVersionNumber()); + } + + eventLogger.logMddDownloadResult( + MddDownloadResult.Code.forNumber(downloadException.getDownloadResultCode().getCode()), + groupDetails.build()); return immediateVoidFuture(); }); } private ListenableFuture<Boolean> subscribeGroup(DataFileGroupInternal dataFileGroup) { - return subscribeGroup(dataFileGroup, /* index = */ 0, dataFileGroup.getFileCount()); + return subscribeGroup(dataFileGroup, /* index= */ 0, dataFileGroup.getFileCount()); } // Because the decision to continue iterating or not depends on the result of the asynchronous @@ -2607,7 +2783,7 @@ public class FileGroupManager { } } - private ListenableFuture<Boolean> isAddedGroupDuplicate( + private ListenableFuture<Optional<Integer>> isAddedGroupDuplicate( GroupKey groupKey, DataFileGroupInternal dataFileGroup) { // Search for a non-downloaded version of this group. GroupKey pendingGroupKey = groupKey.toBuilder().setDownloaded(false).build(); @@ -2623,9 +2799,9 @@ public class FileGroupManager { return transformSequentialAsync( fileGroupsMetadata.read(downloadedGroupKey), downloadedGroup -> { - boolean result = + Optional<Integer> result = (downloadedGroup == null) - ? false + ? Optional.of(0) : areSameGroup(dataFileGroup, downloadedGroup); return immediateFuture(result); }); @@ -2638,38 +2814,41 @@ public class FileGroupManager { * * @param newGroup The new config that we received for the client. * @param prevGroup The old config that we already have for the client. - * @return true if the new config contains an upgrade to any file. + * @return absent if the group is the same, otherwise a code for why the new config isn't the same */ - private static boolean areSameGroup( + private static Optional<Integer> areSameGroup( DataFileGroupInternal newGroup, DataFileGroupInternal prevGroup) { // We do not compare the protos directly and check individual fields because proto.equals // also compares extensions (and unknown fields). // TODO: Consider clearing extensions and then comparing protos. if (prevGroup.getBuildId() != newGroup.getBuildId()) { - return false; + return Optional.of(0); } if (!prevGroup.getVariantId().equals(newGroup.getVariantId())) { - return false; + return Optional.of(0); } if (prevGroup.getFileGroupVersionNumber() != newGroup.getFileGroupVersionNumber()) { - return false; + return Optional.of(0); } if (!hasSameFiles(newGroup, prevGroup)) { - return false; + return Optional.of(0); } if (prevGroup.getStaleLifetimeSecs() != newGroup.getStaleLifetimeSecs()) { - return false; + return Optional.of(0); } if (prevGroup.getExpirationDateSecs() != newGroup.getExpirationDateSecs()) { - return false; + return Optional.of(0); } if (!prevGroup.getDownloadConditions().equals(newGroup.getDownloadConditions())) { - return false; + return Optional.of(0); } if (!prevGroup.getAllowedReadersEnum().equals(newGroup.getAllowedReadersEnum())) { - return false; + return Optional.of(0); } - return true; +// if (!prevGroup.getExperimentInfo().equals(newGroup.getExperimentInfo())) { +// return Optional.of(0); +// } + return Optional.absent(); } /** @@ -2744,10 +2923,6 @@ public class FileGroupManager { groupKeyAndGroup -> { DataFileGroupInternal dataFileGroup = groupKeyAndGroup.dataFileGroup(); - if (dataFileGroup == null) { - return immediateVoidFuture(); - } - for (DataFile dataFile : dataFileGroup.getFileList()) { NewFileKey newFileKey = SharedFilesMetadata.createKeyFromDataFile( @@ -2758,7 +2933,8 @@ public class FileGroupManager { SharedFileMissingException.class, e -> { LogUtil.e("%s: Missing file. Logging and deleting file group.", TAG); - logEventWithDataFileGroup(0, eventLogger, dataFileGroup); + logEventWithDataFileGroup( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, eventLogger, dataFileGroup); if (flags.deleteFileGroupsWithFilesMissing()) { return transformSequentialAsync( @@ -2796,7 +2972,7 @@ public class FileGroupManager { } return transformSequentialAsync( - maybeVerifyIsolatedStructure(dataFileGroup, /*isDownloaded=*/ true), + maybeVerifyIsolatedStructure(dataFileGroup, /* isDownloaded= */ true), verified -> { if (!verified) { return PropagatedFluentFuture.from(createIsolatedFilePaths(dataFileGroup)) @@ -2818,19 +2994,6 @@ public class FileGroupManager { }); } - @AutoValue - abstract static class GroupKeyAndGroup { - static GroupKeyAndGroup create( - GroupKey groupKey, @Nullable DataFileGroupInternal dataFileGroup) { - return new AutoValue_FileGroupManager_GroupKeyAndGroup(groupKey, dataFileGroup); - } - - abstract GroupKey groupKey(); - - @Nullable - abstract DataFileGroupInternal dataFileGroup(); - } - private ListenableFuture<Void> iterateOverAllFileGroups( AsyncFunction<GroupKeyAndGroup, Void> processGroup) { @@ -2844,7 +3007,9 @@ public class FileGroupManager { transformSequentialAsync( fileGroupsMetadata.read(groupKey), dataFileGroup -> - processGroup.apply(GroupKeyAndGroup.create(groupKey, dataFileGroup)))); + (dataFileGroup != null) + ? processGroup.apply(GroupKeyAndGroup.create(groupKey, dataFileGroup)) + : immediateVoidFuture())); } return PropagatedFutures.whenAllComplete(allGroupsProcessed) .call(() -> null, sequentialControlExecutor); @@ -2859,22 +3024,21 @@ public class FileGroupManager { transformSequentialAsync( fileGroupsMetadata.getAllFreshGroups(), dataFileGroups -> { - ArrayList<Pair<GroupKey, DataFileGroupInternal>> sortedFileGroups = - new ArrayList<>(dataFileGroups); + ArrayList<GroupKeyAndGroup> sortedFileGroups = new ArrayList<>(dataFileGroups); Collections.sort( sortedFileGroups, (pairA, pairB) -> ComparisonChain.start() - .compare(pairA.first.getGroupName(), pairB.first.getGroupName()) - .compare(pairA.first.getAccount(), pairB.first.getAccount()) + .compare(pairA.groupKey().getGroupName(), pairB.groupKey().getGroupName()) + .compare(pairA.groupKey().getAccount(), pairB.groupKey().getAccount()) .result()); - for (Pair<GroupKey, DataFileGroupInternal> dataFileGroupPair : sortedFileGroups) { + for (GroupKeyAndGroup dataFileGroupPair : sortedFileGroups) { // TODO(b/131166925): MDD dump should not use lite proto toString. writer.format( "GroupName: %s\nAccount: %s\nDataFileGroup:\n %s\n\n", - dataFileGroupPair.first.getGroupName(), - dataFileGroupPair.first.getAccount(), - dataFileGroupPair.second.toString()); + dataFileGroupPair.groupKey().getGroupName(), + dataFileGroupPair.groupKey().getAccount(), + dataFileGroupPair.dataFileGroup().toString()); } return immediateVoidFuture(); }); @@ -2923,7 +3087,7 @@ public class FileGroupManager { } private static void logEventWithDataFileGroup( - int code, EventLogger eventLogger, DataFileGroupInternal fileGroup) { + MddClientEvent.Code code, EventLogger eventLogger, DataFileGroupInternal fileGroup) { eventLogger.logEventSampled( code, fileGroup.getGroupName(), diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/FileGroupsMetadata.java b/java/com/google/android/libraries/mobiledatadownload/internal/FileGroupsMetadata.java index af555b0..469a09c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/FileGroupsMetadata.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/FileGroupsMetadata.java @@ -15,7 +15,7 @@ */ package com.google.android.libraries.mobiledatadownload.internal; -import android.util.Pair; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup; import com.google.common.util.concurrent.ListenableFuture; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; @@ -71,7 +71,7 @@ public interface FileGroupsMetadata { * @return A future resolving to a list containing pairs of serialized GroupKeys and the * corresponding DataFileGroups. */ - ListenableFuture<List<Pair<GroupKey, DataFileGroupInternal>>> getAllFreshGroups(); + ListenableFuture<List<GroupKeyAndGroup>> getAllFreshGroups(); /** * Removes all entries with a key in keys from the SharedPreferencesFileGroupsMetadata's storage. diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/MddConstants.java b/java/com/google/android/libraries/mobiledatadownload/internal/MddConstants.java index fae84b2..539ac59 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/MddConstants.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/MddConstants.java @@ -58,4 +58,12 @@ public class MddConstants { public static final String SIDELOAD_FILE_URL_SCHEME = "file"; public static final String EMBEDDED_ASSET_URL_SCHEME = "asset"; + + /** + * Currently used in getFileGroup logging. If a matching file group is not found, build_id and + * file_group_version_number are set to below values for logging. + */ + public static final int FILE_GROUP_NOT_FOUND_BUILD_ID = -1; + + public static final int FILE_GROUP_NOT_FOUND_FILE_GROUP_VERSION_NUMBER = -1; } diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/MobileDataDownloadManager.java b/java/com/google/android/libraries/mobiledatadownload/internal/MobileDataDownloadManager.java index 7285ebe..b9496e2 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/MobileDataDownloadManager.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/MobileDataDownloadManager.java @@ -15,6 +15,9 @@ */ package com.google.android.libraries.mobiledatadownload.internal; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.util.concurrent.Futures.getDone; +import static com.google.common.util.concurrent.Futures.immediateFailedFuture; import static com.google.common.util.concurrent.Futures.immediateFuture; import static com.google.common.util.concurrent.Futures.immediateVoidFuture; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; @@ -22,9 +25,6 @@ import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import android.content.Context; import android.content.SharedPreferences; import android.net.Uri; -import android.os.Build.VERSION; -import android.os.Build.VERSION_CODES; -import android.util.Pair; import androidx.annotation.VisibleForTesting; import com.google.android.libraries.mobiledatadownload.FileSource; import com.google.android.libraries.mobiledatadownload.Flags; @@ -33,8 +33,10 @@ import com.google.android.libraries.mobiledatadownload.annotations.InstanceId; import com.google.android.libraries.mobiledatadownload.file.transforms.TransformProtos; import com.google.android.libraries.mobiledatadownload.internal.FileGroupManager.GroupDownloadStatus; import com.google.android.libraries.mobiledatadownload.internal.annotations.SequentialControlExecutor; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup; import com.google.android.libraries.mobiledatadownload.internal.downloader.FileValidator; import com.google.android.libraries.mobiledatadownload.internal.experimentation.DownloadStageManager; +import com.google.android.libraries.mobiledatadownload.internal.logging.DownloadStateLogger; import com.google.android.libraries.mobiledatadownload.internal.logging.EventLogger; import com.google.android.libraries.mobiledatadownload.internal.logging.FileGroupStatsLogger; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; @@ -49,21 +51,21 @@ import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.AsyncFunction; -import com.google.common.util.concurrent.FluentFuture; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.errorprone.annotations.CheckReturnValue; -import com.google.mobiledatadownload.TransformProto.Transforms; import com.google.mobiledatadownload.internal.MetadataProto.DataFile; import com.google.mobiledatadownload.internal.MetadataProto.DataFile.ChecksumType; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions; import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; +import com.google.mobiledatadownload.TransformProto.Transforms; import com.google.protobuf.Any; import java.io.IOException; import java.io.PrintWriter; import java.util.ArrayList; import java.util.List; +import java.util.Map.Entry; import java.util.concurrent.Executor; import javax.annotation.concurrent.NotThreadSafe; import javax.inject.Inject; @@ -167,11 +169,12 @@ public class MobileDataDownloadManager { if (isInitialized) { return immediateVoidFuture(); } - SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_MANAGER_METADATA, instanceId); - return PropagatedFluentFuture.from(Futures.immediateFuture(null)) + return PropagatedFluentFuture.from(immediateVoidFuture()) .transformAsync( voidArg -> { + SharedPreferences prefs = + SharedPreferencesUtil.getSharedPreferences( + context, MDD_MANAGER_METADATA, instanceId); // Offroad downloader migration. Since the migration has been enabled in gms // v18, most devices have migrated. For the remaining, we will clear MDD // storage. @@ -185,7 +188,7 @@ public class MobileDataDownloadManager { }, sequentialControlExecutor); } - return Futures.immediateFuture(null); + return immediateVoidFuture(); }, sequentialControlExecutor) .transformAsync( @@ -195,10 +198,11 @@ public class MobileDataDownloadManager { initSuccess -> { if (!initSuccess) { // This should be init before the shared file metadata. - LogUtil.w("%s Failed to init shared file manager.", TAG); + LogUtil.w( + "%s Clearing MDD since FileManager failed or needs migration.", TAG); return clearForInit(); } - return Futures.immediateVoidFuture(); + return immediateVoidFuture(); }, sequentialControlExecutor), sequentialControlExecutor) @@ -208,10 +212,11 @@ public class MobileDataDownloadManager { sharedFilesMetadata.init(), initSuccess -> { if (!initSuccess) { - LogUtil.w("%s Failed to init shared file metadata.", TAG); + LogUtil.w( + "%s Clearing MDD since FilesMetadata failed or needs migration.", TAG); return clearForInit(); } - return Futures.immediateVoidFuture(); + return immediateVoidFuture(); }, sequentialControlExecutor), sequentialControlExecutor) @@ -243,8 +248,7 @@ public class MobileDataDownloadManager { // instead of boolean for failure public ListenableFuture<Boolean> addGroupForDownload( GroupKey groupKey, DataFileGroupInternal dataFileGroup) { - return addGroupForDownloadInternal( - groupKey, dataFileGroup, unused -> Futures.immediateFuture(true)); + return addGroupForDownloadInternal(groupKey, dataFileGroup, unused -> immediateFuture(true)); } public ListenableFuture<Boolean> addGroupForDownloadInternal( @@ -258,55 +262,93 @@ public class MobileDataDownloadManager { // Check if the group we received is a valid group. if (!DataFileGroupValidator.isValidGroup(dataFileGroup, context, flags)) { eventLogger.logEventSampled( - 0, + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, dataFileGroup.getGroupName(), dataFileGroup.getFileGroupVersionNumber(), dataFileGroup.getBuildId(), dataFileGroup.getVariantId()); - return Futures.immediateFuture(false); + return immediateFuture(false); } DataFileGroupInternal populatedDataFileGroup = mayPopulateChecksum(dataFileGroup); try { - return PropagatedFutures.transformAsync( - fileGroupManager.addGroupForDownload(groupKey, populatedDataFileGroup), - addGroupForDownloadResult -> { - if (addGroupForDownloadResult) { - return PropagatedFutures.transform( - fileGroupManager.verifyPendingGroupDownloaded( - groupKey, populatedDataFileGroup, customFileGroupValidator), - verifyPendingGroupDownloadedResult -> { - if (verifyPendingGroupDownloadedResult - == GroupDownloadStatus.DOWNLOADED) { - eventLogger.logEventSampled( - 0, - populatedDataFileGroup.getGroupName(), - populatedDataFileGroup.getFileGroupVersionNumber(), - populatedDataFileGroup.getBuildId(), - populatedDataFileGroup.getVariantId()); - } - return true; - }, - sequentialControlExecutor); - } - return Futures.immediateFuture(true); - }, - sequentialControlExecutor); + return PropagatedFluentFuture.from( + fileGroupManager.addGroupForDownload(groupKey, populatedDataFileGroup)) + .transformAsync( + addGroupForDownloadResult -> { + if (addGroupForDownloadResult) { + return maybeMarkPendingGroupAsDownloadedImmediately( + groupKey, customFileGroupValidator); + } + return immediateVoidFuture(); + }, + sequentialControlExecutor) + .transform(unused -> true, sequentialControlExecutor); } catch (ExpiredFileGroupException | UninstalledAppException | ActivationRequiredForGroupException e) { LogUtil.w("%s %s", TAG, e.getClass()); - return Futures.immediateFailedFuture(e); + return immediateFailedFuture(e); } catch (IOException e) { LogUtil.e("%s %s", TAG, e.getClass()); silentFeedback.send(e, "Failed to add group to MDD"); - return Futures.immediateFailedFuture(e); + return immediateFailedFuture(e); } }, sequentialControlExecutor); } /** + * Helper method to mark a group as downloaded immediately. + * + * <p>This method checks if a pending group is already downloaded and updates its state in MDD's + * metadata if it is downloaded. Additionally, a download complete immediate event is logged for + * this case. + * + * <p>If no pending version of the group is available, this method is a no-op. + * + * <p>NOTE: This method is only meant to be called during addFileGroup, where it makes sense to + * log the immediate download complete event. + */ + private ListenableFuture<Void> maybeMarkPendingGroupAsDownloadedImmediately( + GroupKey groupKey, AsyncFunction<DataFileGroupInternal, Boolean> customFileGroupValidator) { + ListenableFuture<@NullableType DataFileGroupInternal> pendingGroupFuture = + fileGroupManager.getFileGroup(groupKey, /* downloaded= */ false); + return PropagatedFluentFuture.from(pendingGroupFuture) + .transformAsync( + pendingGroup -> { + if (pendingGroup == null) { + // send pending state to skip logging the event + return immediateFuture(GroupDownloadStatus.PENDING); + } + // Verify the group is downloaded (and commit this to metadata). + return fileGroupManager.verifyGroupDownloaded( + groupKey, + pendingGroup, + /* removePendingVersion= */ true, + customFileGroupValidator, + DownloadStateLogger.forDownload(eventLogger)); + }, + sequentialControlExecutor) + .transformAsync( + verifyPendingGroupDownloadedResult -> { + if (verifyPendingGroupDownloadedResult == GroupDownloadStatus.DOWNLOADED) { + // Use checkNotNull to satisfy nullness checker -- if the group status is + // downloaded, pendingGroup must be non-null. + DataFileGroupInternal group = checkNotNull(getDone(pendingGroupFuture)); + eventLogger.logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + group.getGroupName(), + group.getFileGroupVersionNumber(), + group.getBuildId(), + group.getVariantId()); + } + return immediateVoidFuture(); + }, + sequentialControlExecutor); + } + + /** * Removes the file group from MDD with the given group key. This will cancel any ongoing download * of the file group. * @@ -321,7 +363,7 @@ public class MobileDataDownloadManager { throws SharedFileMissingException, IOException { LogUtil.d("%s removeFileGroup %s", TAG, groupKey.getGroupName()); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( init(), voidArg -> fileGroupManager.removeFileGroup(groupKey, pendingOnly), sequentialControlExecutor); @@ -339,7 +381,7 @@ public class MobileDataDownloadManager { public ListenableFuture<Void> removeFileGroups(List<GroupKey> groupKeys) { LogUtil.d("%s removeFileGroups for %d groups", TAG, groupKeys.size()); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( init(), voidArg -> fileGroupManager.removeFileGroups(groupKeys), sequentialControlExecutor); } @@ -356,65 +398,115 @@ public class MobileDataDownloadManager { GroupKey groupKey, boolean downloaded) { LogUtil.d("%s getFileGroup %s %s", TAG, groupKey.getGroupName(), groupKey.getOwnerPackage()); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( init(), voidArg -> fileGroupManager.getFileGroup(groupKey, downloaded), sequentialControlExecutor); } /** Returns a future resolving to a list of all pending and downloaded groups in MDD. */ - public ListenableFuture<List<Pair<GroupKey, DataFileGroupInternal>>> getAllFreshGroups() { + public ListenableFuture<List<GroupKeyAndGroup>> getAllFreshGroups() { LogUtil.d("%s getAllFreshGroups", TAG); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( init(), voidArg -> fileGroupsMetadata.getAllFreshGroups(), sequentialControlExecutor); } /** - * Returns a future resolving to the URI at which the given data file is located on the disc. - * Returns null if there was error in generating the URI. + * Returns a map of on-device URIs for the requested {@link DataFileGroupInternal}. + * + * <p>If a DataFile does not have an on-device URI (e.g. the download for the file is not + * completed), The returned map will not contain an entry for that DataFile. + * + * <p>If the group supports isolated structures, verification of the isolated structure can be + * controlled. If a file fails the verification (either the symlink is not created, or does not + * point to the correct location), it will be omitted from the map. + * + * <p>NOTE: Verification should only be turned off on critical access paths where latency must be + * minimized. This may lead to an edge case where the isolated structure becomes broken and/or + * corrupted until MDD can fix the structure in its daily maintenance task. */ - public ListenableFuture<@NullableType Uri> getDataFileUri( - DataFile dataFile, DataFileGroupInternal dataFileGroup) { - LogUtil.d("%s getDataFileUri %s %s", TAG, dataFile.getFileId(), dataFileGroup.getGroupName()); - return Futures.transformAsync( - init(), - voidArg -> { - ListenableFuture<@NullableType Uri> onDeviceUriFuture = - fileGroupManager.getOnDeviceUri(dataFile, dataFileGroup); - return Futures.transform( - onDeviceUriFuture, - onDeviceUri -> { - Uri finalOnDeviceUri = onDeviceUri; - // Check if file group should use isolated uri - if (finalOnDeviceUri != null - && FileGroupUtil.isIsolatedStructureAllowed(dataFileGroup) - && VERSION.SDK_INT >= VERSION_CODES.LOLLIPOP) { - try { - finalOnDeviceUri = - fileGroupManager.getAndVerifyIsolatedFileUri( - finalOnDeviceUri, dataFile, dataFileGroup); - } catch (IOException e) { - LogUtil.e( - e, - "%s getDataFileUri %s %s unable to get isolated file uri!", - TAG, - dataFile.getFileId(), - dataFileGroup.getGroupName()); - finalOnDeviceUri = null; - } + public ListenableFuture<ImmutableMap<DataFile, Uri>> getDataFileUris( + DataFileGroupInternal dataFileGroup, boolean verifyIsolatedStructure) { + LogUtil.d("%s: getDataFileUris %s", TAG, dataFileGroup.getGroupName()); + + boolean useIsolatedStructure = FileGroupUtil.isIsolatedStructureAllowed(dataFileGroup); + + // If isolated structure is supported, get the isolated uris (symlinks which point to the + // on-device location). These can be calculated synchronously and before init since they only + // require the file group metadata. + ImmutableMap.Builder<DataFile, Uri> isolatedUriMapBuilder = ImmutableMap.builder(); + if (useIsolatedStructure) { + isolatedUriMapBuilder.putAll(fileGroupManager.getIsolatedFileUris(dataFileGroup)); + } + ImmutableMap<DataFile, Uri> isolatedUriMap = isolatedUriMapBuilder.build(); + + return PropagatedFluentFuture.from(init()) + .transformAsync( + unused -> { + // Lookup on-device uris only if required to reduce latency. On-device lookups happen + // asynchronously since we need to access the latest underlying file metadata. + // 1. The group does not support an isolated structure + // 2. The group supports an isolated structure AND verification of that structure + // should occur. + if (!useIsolatedStructure || verifyIsolatedStructure) { + return fileGroupManager.getOnDeviceUris(dataFileGroup); + } + + // Return an empty map here since we won't be using the on-device uris. + return immediateFuture(ImmutableMap.of()); + }, + sequentialControlExecutor) + .transform( + onDeviceUriMap -> { + if (useIsolatedStructure) { + if (verifyIsolatedStructure) { + // Return verified map of isolated uris. + return fileGroupManager.verifyIsolatedFileUris(isolatedUriMap, onDeviceUriMap); } - if (finalOnDeviceUri != null && dataFile.hasReadTransforms()) { - finalOnDeviceUri = - applyTransformsToFileUri(finalOnDeviceUri, dataFile.getReadTransforms()); + // Verification not required, return isolated uris. + return isolatedUriMap; + } + + // Isolated structure are not in use, return on-device uris. + return onDeviceUriMap; + }, + sequentialControlExecutor) + .transform( + selectedUriMap -> { + // Before returning uri map, apply read transforms if required. + ImmutableMap.Builder<DataFile, Uri> finalUriMapBuilder = ImmutableMap.builder(); + for (Entry<DataFile, Uri> entry : selectedUriMap.entrySet()) { + DataFile dataFile = entry.getKey(); + // Skip entries which have a null uri value. + if (entry.getValue() == null) { + continue; + } + if (dataFile.hasReadTransforms()) { + finalUriMapBuilder.put( + dataFile, + applyTransformsToFileUri(entry.getValue(), dataFile.getReadTransforms())); + } else { + finalUriMapBuilder.put(entry); } + } + return finalUriMapBuilder.build(); + }, + sequentialControlExecutor); + } - return finalOnDeviceUri; - }, - sequentialControlExecutor); - }, - sequentialControlExecutor); + /** + * Convenience method for {@link #getDataFileUris(DataFileGroupInternal, boolean)} when only a + * single data file is required. + */ + public ListenableFuture<@NullableType Uri> getDataFileUri( + DataFile dataFile, DataFileGroupInternal dataFileGroup, boolean verifyIsolatedStructure) { + LogUtil.d("%s getDataFileUri %s %s", TAG, dataFile.getFileId(), dataFileGroup.getGroupName()); + return PropagatedFutures.transform( + getDataFileUris(dataFileGroup, verifyIsolatedStructure), + dataFileUris -> dataFileUris.get(dataFile), + directExecutor()); } private Uri applyTransformsToFileUri(Uri fileUri, Transforms transforms) { @@ -428,7 +520,7 @@ public class MobileDataDownloadManager { } /** - * Import inline files into an exising DataFileGroup and update its metadata accordingly. + * Import inline files into an existing DataFileGroup and update its metadata accordingly. * * @param groupKey The key of file group to update * @param buildId build id to identify the file group to update @@ -448,7 +540,7 @@ public class MobileDataDownloadManager { Optional<Any> customPropertyOptional, AsyncFunction<DataFileGroupInternal, Boolean> customFileGroupValidator) { LogUtil.d("%s: importFiles %s %s", TAG, groupKey.getGroupName(), groupKey.getOwnerPackage()); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( init(), voidArg -> fileGroupManager.importFilesIntoFileGroup( @@ -476,7 +568,7 @@ public class MobileDataDownloadManager { AsyncFunction<DataFileGroupInternal, Boolean> customFileGroupValidator) { LogUtil.d( "%s downloadFileGroup %s %s", TAG, groupKey.getGroupName(), groupKey.getOwnerPackage()); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( init(), voidArg -> fileGroupManager.downloadFileGroup( @@ -494,7 +586,7 @@ public class MobileDataDownloadManager { public ListenableFuture<Boolean> setGroupActivation(GroupKey groupKey, boolean activation) { LogUtil.d( "%s setGroupActivation %s %s", TAG, groupKey.getGroupName(), groupKey.getOwnerPackage()); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( init(), voidArg -> fileGroupManager.setGroupActivation(groupKey, activation), sequentialControlExecutor); @@ -509,11 +601,11 @@ public class MobileDataDownloadManager { public ListenableFuture<Void> downloadAllPendingGroups( boolean onWifi, AsyncFunction<DataFileGroupInternal, Boolean> customFileGroupValidator) { LogUtil.d("%s downloadAllPendingGroups on wifi = %s", TAG, onWifi); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( init(), voidArg -> { if (flags.mddEnableDownloadPendingGroups()) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return fileGroupManager.scheduleAllPendingGroupsForDownload( onWifi, customFileGroupValidator); } @@ -529,11 +621,11 @@ public class MobileDataDownloadManager { public ListenableFuture<Void> verifyAllPendingGroups( AsyncFunction<DataFileGroupInternal, Boolean> customFileGroupValidator) { LogUtil.d("%s verifyAllPendingGroups", TAG); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( init(), voidArg -> { if (flags.mddEnableVerifyPendingGroups()) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return fileGroupManager.verifyAllPendingGroupsDownloaded(customFileGroupValidator); } return immediateVoidFuture(); @@ -552,7 +644,7 @@ public class MobileDataDownloadManager { public ListenableFuture<Void> maintenance() { LogUtil.d("%s Running maintenance", TAG); - return FluentFuture.from(init()) + return PropagatedFluentFuture.from(init()) .transformAsync(voidArg -> getAndResetDaysSinceLastMaintenance(), directExecutor()) .transformAsync( daysSinceLastLog -> { @@ -582,7 +674,7 @@ public class MobileDataDownloadManager { if (flags.mddEnableGarbageCollection()) { maintenanceFutures.add(expirationHandler.updateExpiration()); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); } // Log daily file group stats. @@ -601,18 +693,27 @@ public class MobileDataDownloadManager { context, MDD_MANAGER_METADATA, instanceId); prefs.edit().remove(MDD_PH_CONFIG_VERSION).remove(MDD_PH_CONFIG_VERSION_TS).commit(); - return Futures.whenAllComplete(maintenanceFutures) + return PropagatedFutures.whenAllComplete(maintenanceFutures) .call(() -> null, sequentialControlExecutor); }, sequentialControlExecutor); } + /** + * Removes expired FileGroups (whether active or stale) and deletes files no longer referenced by + * a FileGroup. + */ + public ListenableFuture<Void> removeExpiredGroupsAndFiles() { + return PropagatedFluentFuture.from(init()) + .transformAsync(voidArg -> expirationHandler.updateExpiration(), sequentialControlExecutor); + } + /** Dumps the current internal state of the MDD manager. */ public ListenableFuture<Void> dump(final PrintWriter writer) { - return Futures.transformAsync( + return PropagatedFutures.transformAsync( init(), voidArg -> - Futures.transformAsync( + PropagatedFutures.transformAsync( fileGroupManager.dump(writer), voidParam -> sharedFileManager.dump(writer), sequentialControlExecutor), @@ -622,7 +723,7 @@ public class MobileDataDownloadManager { /** Checks to see if a flag change requires MDD to clear its data. */ public ListenableFuture<Void> checkResetTrigger() { LogUtil.d("%s checkResetTrigger", TAG); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( init(), voidArg -> { SharedPreferences prefs = @@ -637,7 +738,7 @@ public class MobileDataDownloadManager { if (savedResetValue < currentResetValue) { prefs.edit().putInt(RESET_TRIGGER, currentResetValue).commit(); LogUtil.d("%s Received reset trigger. Clearing all Mdd data.", TAG); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return clearAllFilesAndMetadata(); } return immediateVoidFuture(); @@ -692,12 +793,12 @@ public class MobileDataDownloadManager { /* Clear all metadata and files, also cancel pending download. */ private ListenableFuture<Void> clearAllFilesAndMetadata() { - return Futures.transformAsync( + return PropagatedFutures.transformAsync( // Need to cancel download after MDD is already initialized. sharedFileManager.cancelDownloadAndClear(), voidArg1 -> // The metadata files should be cleared after the classes have been cleared. - Futures.transformAsync( + PropagatedFutures.transformAsync( sharedFilesMetadata.clear(), voidArg2 -> fileGroupsMetadata.clear(), sequentialControlExecutor), @@ -772,7 +873,7 @@ public class MobileDataDownloadManager { return immediateFuture(DEFAULT_DAYS_SINCE_LAST_MAINTENANCE); } - return FluentFuture.from(loggingStateStore.getAndResetDaysSinceLastMaintenance()) + return PropagatedFluentFuture.from(loggingStateStore.getAndResetDaysSinceLastMaintenance()) .catching( IOException.class, exception -> { diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/SharedFileManager.java b/java/com/google/android/libraries/mobiledatadownload/internal/SharedFileManager.java index c0804b7..2c1775d 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/SharedFileManager.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/SharedFileManager.java @@ -15,7 +15,11 @@ */ package com.google.android.libraries.mobiledatadownload.internal; +import static com.google.common.util.concurrent.Futures.getDone; +import static com.google.common.util.concurrent.Futures.immediateFailedFuture; +import static com.google.common.util.concurrent.Futures.immediateFuture; import static com.google.common.util.concurrent.Futures.immediateVoidFuture; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import android.content.Context; import android.content.SharedPreferences; @@ -45,8 +49,11 @@ import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.android.libraries.mobiledatadownload.internal.util.DirectoryUtil; import com.google.android.libraries.mobiledatadownload.internal.util.SharedPreferencesUtil; import com.google.android.libraries.mobiledatadownload.monitor.DownloadProgressMonitor; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentFuture; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Optional; -import com.google.common.util.concurrent.FluentFuture; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.errorprone.annotations.CheckReturnValue; @@ -61,6 +68,7 @@ import com.google.mobiledatadownload.internal.MetadataProto.FileStatus; import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey; import com.google.mobiledatadownload.internal.MetadataProto.SharedFile; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; import java.io.IOException; import java.io.PrintWriter; import java.util.ArrayList; @@ -166,7 +174,7 @@ public class SharedFileManager { sharedFileManagerMetadata.edit().remove(PREFS_KEY_MIGRATED_TO_NEW_FILE_KEY).commit(); } - return Futures.immediateFuture(true); + return immediateFuture(true); } /** @@ -178,12 +186,12 @@ public class SharedFileManager { */ // TODO - refactor to throw Exception when write to SharedPreferences fails public ListenableFuture<Boolean> reserveFileEntry(NewFileKey newFileKey) { - return Futures.transformAsync( + return PropagatedFutures.transformAsync( sharedFilesMetadata.read(newFileKey), sharedFile -> { if (sharedFile != null) { // There's already an entry for this file. Nothing to do here. - return Futures.immediateFuture(true); + return immediateFuture(true); } // Set the file name and update the metadata file. SharedPreferences sharedFileManagerMetadata = @@ -198,7 +206,7 @@ public class SharedFileManager { .commit()) { // TODO(b/131166925): MDD dump should not use lite proto toString. LogUtil.e("%s: Unable to update file name %s", TAG, newFileKey); - return Futures.immediateFuture(false); + return immediateFuture(false); } String fileName = FILE_NAME_PREFIX + nextFileName; @@ -207,7 +215,7 @@ public class SharedFileManager { .setFileStatus(FileStatus.SUBSCRIBED) .setFileName(fileName) .build(); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( sharedFilesMetadata.write(newFileKey, sharedFile), writeSuccess -> { if (!writeSuccess) { @@ -215,9 +223,9 @@ public class SharedFileManager { LogUtil.e( "%s: Unable to write back subscription for file entry with %s", TAG, newFileKey); - return Futures.immediateFuture(false); + return immediateFuture(false); } - return Futures.immediateFuture(true); + return immediateFuture(true); }, sequentialControlExecutor); }, @@ -239,13 +247,13 @@ public class SharedFileManager { @Nullable DownloadConditions downloadConditions, FileSource inlineFileSource) { if (!dataFile.getUrlToDownload().startsWith(MddConstants.INLINE_FILE_URL_SCHEME)) { - return Futures.immediateFailedFuture( + return immediateFailedFuture( DownloadException.builder() .setDownloadResultCode(DownloadResultCode.INVALID_INLINE_FILE_URL_SCHEME) .setMessage("Importing an inline file requires inlinefile scheme") .build()); } - return Futures.transformAsync( + return PropagatedFutures.transformAsync( sharedFilesMetadata.read(newFileKey), sharedFile -> { if (sharedFile == null) { @@ -254,7 +262,7 @@ public class SharedFileManager { TAG, dataFile.getFileId()); SharedFileMissingException cause = new SharedFileMissingException(); // TODO(b/167582815): Log to Clearcut - return Futures.immediateFailedFuture( + return immediateFailedFuture( DownloadException.builder() .setDownloadResultCode(DownloadResultCode.SHARED_FILE_NOT_FOUND_ERROR) .setCause(cause) @@ -274,7 +282,7 @@ public class SharedFileManager { sharedFile.getFileName(), dataFile.getDownloadedFileChecksum()) : sharedFile.getFileName(); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( getDataFileGroupOrDefault(groupKey), dataFileGroup -> getImportFuture( @@ -313,7 +321,7 @@ public class SharedFileManager { ListenableFuture<Uri> downloadFileOnDeviceUriFuture = getDownloadFileOnDeviceUri( newFileKey.getAllowedReaders(), downloadFileName, dataFile.getChecksum()); - return FluentFuture.from(downloadFileOnDeviceUriFuture) + return PropagatedFluentFuture.from(downloadFileOnDeviceUriFuture) .transformAsync( unused -> { sharedFileBuilder.setFileStatus(FileStatus.DOWNLOAD_IN_PROGRESS); @@ -327,7 +335,7 @@ public class SharedFileManager { sequentialControlExecutor) .transformAsync( unused -> { - Uri downloadFileOnDeviceUri = Futures.getDone(downloadFileOnDeviceUriFuture); + Uri downloadFileOnDeviceUri = getDone(downloadFileOnDeviceUriFuture); DownloaderCallback downloaderCallback = new DownloaderCallbackImpl( sharedFilesMetadata, @@ -345,6 +353,7 @@ public class SharedFileManager { // progress here. return fileDownloader.startCopying( + newFileKey.getChecksum(), downloadFileOnDeviceUri, dataFile.getUrlToDownload(), dataFile.getByteSize(), @@ -376,7 +385,7 @@ public class SharedFileManager { int trafficTag, List<ExtraHttpHeader> extraHttpHeaders) { if (dataFile.getUrlToDownload().startsWith(MddConstants.INLINE_FILE_URL_SCHEME)) { - return Futures.immediateFailedFuture( + return immediateFailedFuture( DownloadException.builder() .setDownloadResultCode(DownloadResultCode.INVALID_INLINE_FILE_URL_SCHEME) .setMessage( @@ -384,77 +393,134 @@ public class SharedFileManager { + " instead.") .build()); } - return Futures.transformAsync( - sharedFilesMetadata.read(newFileKey), - sharedFile -> { - if (sharedFile == null) { - // TODO(b/131166925): MDD dump should not use lite proto toString. - LogUtil.e( - "%s: Start download called on file that doesn't exists. Key = %s!", - TAG, newFileKey); - SharedFileMissingException cause = new SharedFileMissingException(); - silentFeedback.send(cause, "Shared file not found in downloadFileGroup"); - return Futures.immediateFailedFuture( - DownloadException.builder() - .setDownloadResultCode(DownloadResultCode.SHARED_FILE_NOT_FOUND_ERROR) - .setCause(cause) - .build()); - } - // If we have already downloaded the file, then return. - if (sharedFile.getFileStatus() == FileStatus.DOWNLOAD_COMPLETE) { - if (downloadMonitorOptional.isPresent()) { - // For the downloaded file, we don't need to monitor the file change. We just need to - // inform the monitor about its current size. - downloadMonitorOptional - .get() - .notifyCurrentFileSize(groupKey.getGroupName(), dataFile.getByteSize()); - } - return immediateVoidFuture(); - } + // Start futures in parallel for various calculated properties. + ListenableFuture<SharedFile> sharedFileFuture = getSharedFile(newFileKey); + + ListenableFuture<@NullableType DeltaFile> firstDeltaFileFuture = + findFirstDeltaFileWithBaseFileDownloaded(dataFile, newFileKey.getAllowedReaders()); + + ListenableFuture<String> downloadFileNameFuture = + PropagatedFutures.whenAllSucceed(sharedFileFuture, firstDeltaFileFuture) + .call( + () -> { + String downloadFileName = getDone(sharedFileFuture).getFileName(); + DeltaFile deltaFile = getDone(firstDeltaFileFuture); + if (deltaFile != null) { + downloadFileName = + FileNameUtil.getTempFileNameWithDownloadedFileChecksum( + downloadFileName, deltaFile.getChecksum()); + } else if (dataFile.hasDownloadTransforms()) { + downloadFileName = + FileNameUtil.getTempFileNameWithDownloadedFileChecksum( + downloadFileName, dataFile.getDownloadedFileChecksum()); + } + return downloadFileName; + }, + directExecutor()); + + ListenableFuture<Uri> downloadFileOnDeviceUriFuture = + PropagatedFutures.transformAsync( + downloadFileNameFuture, + downloadFileName -> + getDownloadFileOnDeviceUri( + newFileKey.getAllowedReaders(), downloadFileName, dataFile.getChecksum()), + sequentialControlExecutor); + + ListenableFuture<DataFileGroupInternal> dataFileGroupFuture = + getDataFileGroupOrDefault(groupKey); + + // Combine all futures together so all complete successfully before continuing + ListenableFuture<Void> combinedPropertiesFuture = + PropagatedFutures.whenAllSucceed( + sharedFileFuture, + firstDeltaFileFuture, + downloadFileNameFuture, + downloadFileOnDeviceUriFuture, + dataFileGroupFuture) + .callAsync(Futures::immediateVoidFuture, directExecutor()); + + return PropagatedFluentFuture.from(combinedPropertiesFuture) + .transformAsync( + unused -> { + SharedFile sharedFile = getDone(sharedFileFuture); + DeltaFile deltaFile = getDone(firstDeltaFileFuture); + String downloadFileName = getDone(downloadFileNameFuture); + Uri downloadFileOnDeviceUri = getDone(downloadFileOnDeviceUriFuture); + DataFileGroupInternal dataFileGroup = getDone(dataFileGroupFuture); - return Futures.transformAsync( - findFirstDeltaFileWithBaseFileDownloaded(dataFile, newFileKey.getAllowedReaders()), - deltaFile -> { - SharedFile.Builder sharedFileBuilder = sharedFile.toBuilder(); - String downloadFileName = sharedFile.getFileName(); - if (deltaFile != null) { - downloadFileName = - FileNameUtil.getTempFileNameWithDownloadedFileChecksum( - downloadFileName, deltaFile.getChecksum()); - } else if (dataFile.hasDownloadTransforms()) { - downloadFileName = - FileNameUtil.getTempFileNameWithDownloadedFileChecksum( - downloadFileName, dataFile.getDownloadedFileChecksum()); + // Check if download is complete + if (sharedFile.getFileStatus() == FileStatus.DOWNLOAD_COMPLETE) { + if (downloadMonitorOptional.isPresent()) { + // For the downloaded file, we don't need to monitor the file change. We just need + // to inform the monitor about its current size. + downloadMonitorOptional + .get() + .notifyCurrentFileSize(groupKey.getGroupName(), dataFile.getByteSize()); } + return immediateVoidFuture(); + } - // Variables captured in lambdas must be effectively final. - String downloadFileNameCapture = downloadFileName; - return Futures.transformAsync( - getDataFileGroupOrDefault(groupKey), - dataFileGroup -> - getDownloadFuture( - sharedFileBuilder, - newFileKey, - downloadFileNameCapture, - dataFileGroup.getFileGroupVersionNumber(), - dataFileGroup.getBuildId(), - dataFileGroup.getVariantId(), - groupKey, - dataFile, - deltaFile, - downloadConditions, - trafficTag, - extraHttpHeaders), + // Check if a download is already in progress + if (sharedFile.getFileStatus() == FileStatus.DOWNLOAD_IN_PROGRESS) { + return PropagatedFutures.transformAsync( + fileDownloader.getInProgressFuture( + newFileKey.getChecksum(), downloadFileOnDeviceUri), + inProgressFuture -> { + if (inProgressFuture.isPresent()) { + mayNotifyCurrentSizeOfPartiallyDownloadedFile( + groupKey, downloadFileOnDeviceUri); + return inProgressFuture.get(); + } + return getDownloadFuture( + newFileKey, + downloadFileName, + dataFileGroup.getFileGroupVersionNumber(), + dataFileGroup.getBuildId(), + dataFileGroup.getVariantId(), + groupKey, + dataFile, + deltaFile, + downloadConditions, + trafficTag, + extraHttpHeaders); + }, sequentialControlExecutor); - }, - sequentialControlExecutor); - }, - sequentialControlExecutor); + } + + // Download is not in progress, start it. + return getDownloadFuture( + newFileKey, + downloadFileName, + dataFileGroup.getFileGroupVersionNumber(), + dataFileGroup.getBuildId(), + dataFileGroup.getVariantId(), + groupKey, + dataFile, + deltaFile, + downloadConditions, + trafficTag, + extraHttpHeaders); + }, + sequentialControlExecutor) + .catchingAsync( + SharedFileMissingException.class, + ex -> { + // TODO(b/131166925): MDD dump should not use lite proto toString. + LogUtil.e( + "%s: Start download called on file that doesn't exist. Key = %s!", + TAG, newFileKey); + silentFeedback.send(ex, "Shared file not found in downloadFileGroup"); + return immediateFailedFuture( + DownloadException.builder() + .setDownloadResultCode(DownloadResultCode.SHARED_FILE_NOT_FOUND_ERROR) + .setCause(ex) + .build()); + }, + sequentialControlExecutor); } private ListenableFuture<Void> getDownloadFuture( - SharedFile.Builder sharedFileBuilder, NewFileKey newFileKey, String downloadFileName, int fileGroupVersionNumber, @@ -466,91 +532,114 @@ public class SharedFileManager { @Nullable DownloadConditions downloadConditions, int trafficTag, List<ExtraHttpHeader> extraHttpHeaders) { - ListenableFuture<Uri> downloadFileOnDeviceUriFuture = - getDownloadFileOnDeviceUri( - newFileKey.getAllowedReaders(), downloadFileName, dataFile.getChecksum()); - return FluentFuture.from(downloadFileOnDeviceUriFuture) - .transformAsync( - unused -> { - sharedFileBuilder.setFileStatus(FileStatus.DOWNLOAD_IN_PROGRESS); + // It's possible to hit a race condition where the caller of this method sees the file as not + // downloaded and by the time this method is executed, the file is already downloaded. + // + // Check the shared file status before starting the download to confirm it is not downloaded and + // a download is not already in progress. + return PropagatedFutures.transformAsync( + getSharedFile(newFileKey), + latestSharedFile -> { + if (latestSharedFile.getFileStatus() == FileStatus.DOWNLOAD_COMPLETE) { + return immediateVoidFuture(); + } - // Ignoring failure to write back here, as it will just result in one extra try to - // download the file. - return sharedFilesMetadata.write(newFileKey, sharedFileBuilder.build()); - }, - sequentialControlExecutor) - .transformAsync( - unused -> { - Uri downloadFileOnDeviceUri = Futures.getDone(downloadFileOnDeviceUriFuture); - ListenableFuture<Void> fileDownloadFuture; - if (!deltaDecoderOptional.isPresent() || deltaFile == null) { - // Download full file when delta file is null - DownloaderCallback downloaderCallback = - new DownloaderCallbackImpl( - sharedFilesMetadata, - fileStorage, - dataFile, - newFileKey.getAllowedReaders(), - eventLogger, - groupKey, - fileGroupVersionNumber, - buildId, - variantId, - flags, - sequentialControlExecutor); + // Download is not complete, proceed with starting the future. + SharedFile.Builder sharedFileBuilder = latestSharedFile.toBuilder(); + ListenableFuture<Uri> downloadFileOnDeviceUriFuture = + getDownloadFileOnDeviceUri( + newFileKey.getAllowedReaders(), downloadFileName, dataFile.getChecksum()); + return PropagatedFluentFuture.from(downloadFileOnDeviceUriFuture) + .transformAsync( + unused -> { + sharedFileBuilder.setFileStatus(FileStatus.DOWNLOAD_IN_PROGRESS); - mayNotifyCurrentSizeOfPartiallyDownloadedFile(groupKey, downloadFileOnDeviceUri); + // Ignoring failure to write back here, as it will just result in one + // extra try + // to download the file. + return sharedFilesMetadata.write(newFileKey, sharedFileBuilder.build()); + }, + sequentialControlExecutor) + .transformAsync( + unused -> { + Uri downloadFileOnDeviceUri = getDone(downloadFileOnDeviceUriFuture); + ListenableFuture<Void> fileDownloadFuture; + if (!deltaDecoderOptional.isPresent() || deltaFile == null) { + // Download full file when delta file is null + DownloaderCallback downloaderCallback = + new DownloaderCallbackImpl( + sharedFilesMetadata, + fileStorage, + dataFile, + newFileKey.getAllowedReaders(), + eventLogger, + groupKey, + fileGroupVersionNumber, + buildId, + variantId, + flags, + sequentialControlExecutor); - fileDownloadFuture = - fileDownloader.startDownloading( - groupKey, - fileGroupVersionNumber, - buildId, - downloadFileOnDeviceUri, - dataFile.getUrlToDownload(), - dataFile.getByteSize(), - downloadConditions, - downloaderCallback, - trafficTag, - extraHttpHeaders); - } else { - DownloaderCallback downloaderCallback = - new DeltaFileDownloaderCallbackImpl( - context, - sharedFilesMetadata, - fileStorage, - silentFeedback, - dataFile, - newFileKey.getAllowedReaders(), - deltaDecoderOptional.get(), - deltaFile, - eventLogger, - groupKey, - fileGroupVersionNumber, - buildId, - variantId, - instanceId, - flags, - sequentialControlExecutor); + mayNotifyCurrentSizeOfPartiallyDownloadedFile( + groupKey, downloadFileOnDeviceUri); - mayNotifyCurrentSizeOfPartiallyDownloadedFile(groupKey, downloadFileOnDeviceUri); + fileDownloadFuture = + fileDownloader.startDownloading( + newFileKey.getChecksum(), + groupKey, + fileGroupVersionNumber, + buildId, + variantId, + downloadFileOnDeviceUri, + dataFile.getUrlToDownload(), + dataFile.getByteSize(), + downloadConditions, + downloaderCallback, + trafficTag, + extraHttpHeaders); + } else { + DownloaderCallback downloaderCallback = + new DeltaFileDownloaderCallbackImpl( + context, + sharedFilesMetadata, + fileStorage, + silentFeedback, + dataFile, + newFileKey.getAllowedReaders(), + deltaDecoderOptional.get(), + deltaFile, + eventLogger, + groupKey, + fileGroupVersionNumber, + buildId, + variantId, + instanceId, + flags, + sequentialControlExecutor); - fileDownloadFuture = - fileDownloader.startDownloading( - groupKey, - fileGroupVersionNumber, - buildId, - downloadFileOnDeviceUri, - deltaFile.getUrlToDownload(), - deltaFile.getByteSize(), - downloadConditions, - downloaderCallback, - trafficTag, - extraHttpHeaders); - } - return fileDownloadFuture; - }, - sequentialControlExecutor); + mayNotifyCurrentSizeOfPartiallyDownloadedFile( + groupKey, downloadFileOnDeviceUri); + + fileDownloadFuture = + fileDownloader.startDownloading( + newFileKey.getChecksum(), + groupKey, + fileGroupVersionNumber, + buildId, + variantId, + downloadFileOnDeviceUri, + deltaFile.getUrlToDownload(), + deltaFile.getByteSize(), + downloadConditions, + downloaderCallback, + trafficTag, + extraHttpHeaders); + } + return fileDownloadFuture; + }, + sequentialControlExecutor); + }, + sequentialControlExecutor); } /** @@ -570,15 +659,15 @@ public class SharedFileManager { checksum, silentFeedback, instanceId, - /* androidShared = */ false); + /* androidShared= */ false); if (downloadFileOnDeviceUri == null) { LogUtil.e("%s: Failed to get file uri!", TAG); - return Futures.immediateFailedFuture( + return immediateFailedFuture( DownloadException.builder() .setDownloadResultCode(DownloadResultCode.UNABLE_TO_CREATE_FILE_URI_ERROR) .build()); } - return Futures.immediateFuture(downloadFileOnDeviceUri); + return immediateFuture(downloadFileOnDeviceUri); } private void mayNotifyCurrentSizeOfPartiallyDownloadedFile( @@ -599,10 +688,10 @@ public class SharedFileManager { } private ListenableFuture<DataFileGroupInternal> getDataFileGroupOrDefault(GroupKey groupKey) { - return Futures.transformAsync( + return PropagatedFutures.transformAsync( fileGroupsMetadata.read(groupKey), fileGroup -> - Futures.immediateFuture( + immediateFuture( (fileGroup == null) ? DataFileGroupInternal.getDefaultInstance() : fileGroup), sequentialControlExecutor); } @@ -621,10 +710,10 @@ public class SharedFileManager { < FileKeyVersion.USE_CHECKSUM_ONLY.value || !deltaDecoderOptional.isPresent() || deltaDecoderOptional.get().getDecoderName() == DiffDecoder.UNSPECIFIED) { - return Futures.immediateFuture(null); + return immediateFuture(null); } return findFirstDeltaFileWithBaseFileDownloaded( - dataFile.getDeltaFileList(), /* index = */ 0, allowedReaders); + dataFile.getDeltaFileList(), /* index= */ 0, allowedReaders); } // We must use recursion here since the decision to continue iterating is dependent on the result @@ -632,7 +721,7 @@ public class SharedFileManager { private ListenableFuture<@NullableType DeltaFile> findFirstDeltaFileWithBaseFileDownloaded( List<DeltaFile> deltaFiles, int index, AllowedReaders allowedReaders) { if (index == deltaFiles.size()) { - return Futures.immediateFuture(null); + return immediateFuture(null); } DeltaFile deltaFile = deltaFiles.get(index); if (deltaFile.getDiffDecoder() != deltaDecoderOptional.get().getDecoderName()) { @@ -643,7 +732,7 @@ public class SharedFileManager { .setChecksum(deltaFile.getBaseFile().getChecksum()) .setAllowedReaders(allowedReaders) .build(); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( sharedFilesMetadata.read(baseFileKey), baseFileMetadata -> { if (baseFileMetadata != null @@ -656,9 +745,9 @@ public class SharedFileManager { baseFileKey.getChecksum(), silentFeedback, instanceId, - /* androidShared = */ false); + /* androidShared= */ false); if (baseFileUri != null) { - return Futures.immediateFuture(deltaFile); + return immediateFuture(deltaFile); } } return findFirstDeltaFileWithBaseFileDownloaded(deltaFiles, index + 1, allowedReaders); @@ -673,9 +762,9 @@ public class SharedFileManager { * a SharedFileMissingException if the shared file metadata is missing. */ ListenableFuture<FileStatus> getFileStatus(NewFileKey newFileKey) { - return Futures.transformAsync( + return PropagatedFutures.transformAsync( getSharedFile(newFileKey), - existingSharedFile -> Futures.immediateFuture(existingSharedFile.getFileStatus()), + existingSharedFile -> immediateFuture(existingSharedFile.getFileStatus()), sequentialControlExecutor); } @@ -688,14 +777,14 @@ public class SharedFileManager { * metadata is missing or the on disk file is corrupted. */ ListenableFuture<Void> reVerifyFile(NewFileKey newFileKey, DataFile dataFile) { - return FluentFuture.from(getSharedFile(newFileKey)) + return PropagatedFluentFuture.from(getSharedFile(newFileKey)) .transformAsync( existingSharedFile -> { if (existingSharedFile.getFileStatus() != FileStatus.DOWNLOAD_COMPLETE) { - return Futures.immediateVoidFuture(); + return immediateVoidFuture(); } // Double check that it's really complete, and update status if it's not. - return FluentFuture.from(getOnDeviceUri(newFileKey)) + return PropagatedFluentFuture.from(getOnDeviceUri(newFileKey)) .transformAsync( uri -> { if (uri == null) { @@ -717,7 +806,7 @@ public class SharedFileManager { FileValidator.validateDownloadedFile( fileStorage, dataFile, uri, dataFile.getChecksum()); } - return Futures.immediateVoidFuture(); + return immediateVoidFuture(); }, sequentialControlExecutor) .catchingAsync( @@ -730,7 +819,7 @@ public class SharedFileManager { existingSharedFile.toBuilder() .setFileStatus(FileStatus.CORRUPTED) .build(); - return FluentFuture.from( + return PropagatedFluentFuture.from( sharedFilesMetadata.write(newFileKey, updatedSharedFile)) .transformAsync( ok -> { @@ -755,16 +844,16 @@ public class SharedFileManager { * may throw a SharedFileMissingException if the shared file metadata is missing. */ ListenableFuture<SharedFile> getSharedFile(NewFileKey newFileKey) { - return Futures.transformAsync( + return PropagatedFutures.transformAsync( sharedFilesMetadata.read(newFileKey), existingSharedFile -> { if (existingSharedFile == null) { // TODO(b/131166925): MDD dump should not use lite proto toString. LogUtil.e( - "%s: getSharedFile called on file that doesn't exists! Key = %s", TAG, newFileKey); - return Futures.immediateFailedFuture(new SharedFileMissingException()); + "%s: getSharedFile called on file that doesn't exist! Key = %s", TAG, newFileKey); + return immediateFailedFuture(new SharedFileMissingException()); } - return Futures.immediateFuture(existingSharedFile); + return immediateFuture(existingSharedFile); }, sequentialControlExecutor); } @@ -803,7 +892,7 @@ public class SharedFileManager { */ ListenableFuture<Boolean> updateMaxExpirationDateSecs( NewFileKey newFileKey, long fileExpirationDateSecs) { - return Futures.transformAsync( + return PropagatedFutures.transformAsync( getSharedFile(newFileKey), existingSharedFile -> { if (fileExpirationDateSecs > existingSharedFile.getMaxExpirationDateSecs()) { @@ -813,7 +902,7 @@ public class SharedFileManager { .build(); return sharedFilesMetadata.write(newFileKey, updatedSharedFile); } - return Futures.immediateFuture(true); + return immediateFuture(true); }, sequentialControlExecutor); } @@ -827,29 +916,56 @@ public class SharedFileManager { * is an error populating the uri of the file. */ public ListenableFuture<@NullableType Uri> getOnDeviceUri(NewFileKey newFileKey) { - return Futures.transformAsync( - sharedFilesMetadata.read(newFileKey), - sharedFile -> { - if (sharedFile == null) { - // TODO(b/131166925): MDD dump should not use lite proto toString. - LogUtil.e( - "%s: getOnDeviceUri called on file that doesn't exists. Key = %s!", - TAG, newFileKey); - return Futures.immediateFailedFuture(new SharedFileMissingException()); - } + return PropagatedFutures.transform( + getOnDeviceUris(ImmutableSet.of(newFileKey)), + uris -> uris.get(newFileKey), + directExecutor()); + } - Uri onDeviceUri = - DirectoryUtil.getOnDeviceUri( - context, - newFileKey.getAllowedReaders(), - sharedFile.getFileName(), - sharedFile.getAndroidSharingChecksum(), - silentFeedback, - instanceId, - sharedFile.getAndroidShared()); - return Futures.immediateFuture(onDeviceUri); - }, - sequentialControlExecutor); + /** + * Get the known on-device uris for a given list of {@link NewFileKey}s + * + * <p>The returned map may or may not have an entry for each NewFileKey on the list, depending on + * if it was possible to create the uri (see {@link DirectoryUtil#getOnDeviceUri()} for more + * details). + * + * <p>If any {@link NewFileKey} does not map to a {@link SharedFile}, the returned future will be + * a failure containing {@link SharedFileMissingException}. + */ + ListenableFuture<ImmutableMap<NewFileKey, Uri>> getOnDeviceUris( + ImmutableSet<NewFileKey> newFileKeys) { + return PropagatedFluentFuture.from(sharedFilesMetadata.readAll(newFileKeys)) + .transformAsync( + sharedFileMap -> { + ImmutableMap.Builder<NewFileKey, Uri> uriMapBuilder = ImmutableMap.builder(); + for (NewFileKey newFileKey : newFileKeys) { + // Make sure all SharedFiles exist. + if (!sharedFileMap.containsKey(newFileKey)) { + // TODO(b/131166925): MDD dump should not use lite proto toString. + LogUtil.e( + "%s: getOnDeviceUris called on file that doesn't exist. Key = %s!", + TAG, newFileKey); + return immediateFailedFuture(new SharedFileMissingException()); + } + + SharedFile sharedFile = sharedFileMap.get(newFileKey); + + Uri onDeviceUri = + DirectoryUtil.getOnDeviceUri( + context, + newFileKey.getAllowedReaders(), + sharedFile.getFileName(), + sharedFile.getAndroidSharingChecksum(), + silentFeedback, + instanceId, + sharedFile.getAndroidShared()); + if (onDeviceUri != null) { + uriMapBuilder.put(newFileKey, onDeviceUri); + } + } + return immediateFuture(uriMapBuilder.build()); + }, + sequentialControlExecutor); } /** @@ -861,13 +977,13 @@ public class SharedFileManager { */ // TODO - refactor to throw Exception when write to SharedPreferences fails ListenableFuture<Boolean> removeFileEntry(NewFileKey newFileKey) { - return Futures.transformAsync( + return PropagatedFutures.transformAsync( sharedFilesMetadata.read(newFileKey), sharedFile -> { if (sharedFile == null) { // TODO(b/131166925): MDD dump should not use lite proto toString. LogUtil.e("%s: No file entry with key %s", TAG, newFileKey); - return Futures.immediateFuture(false); + return immediateFuture(false); } Uri onDeviceUri = @@ -878,19 +994,19 @@ public class SharedFileManager { newFileKey.getChecksum(), silentFeedback, instanceId, - /* androidShared = */ false); + /* androidShared= */ false); if (onDeviceUri != null) { - fileDownloader.stopDownloading(onDeviceUri); + fileDownloader.stopDownloading(newFileKey.getChecksum(), onDeviceUri); } - return Futures.transformAsync( + return PropagatedFutures.transformAsync( sharedFilesMetadata.remove(newFileKey), removeSuccess -> { if (!removeSuccess) { // TODO(b/131166925): MDD dump should not use lite proto toString. LogUtil.e("%s: Unable to modify file subscription for key %s", TAG, newFileKey); - return Futures.immediateFuture(false); + return immediateFuture(false); } - return Futures.immediateFuture(true); + return immediateFuture(true); }, sequentialControlExecutor); }, @@ -901,6 +1017,7 @@ public class SharedFileManager { * Clears all storage used by the SharedFileManager and deletes all files that have been * downloaded to MDD's directory. */ + // TODO(b/124072754): Change to package private once all code is refactored. public ListenableFuture<Void> clear() { // If sdk is R+, try release all leases that the MDD Client may have acquired. This @@ -913,14 +1030,14 @@ public class SharedFileManager { } catch (IOException e) { silentFeedback.send(e, "Failure while deleting mdd storage during clear"); } - return Futures.immediateVoidFuture(); + return immediateVoidFuture(); } private void releaseAllAndroidSharedFiles() { try { Uri allLeasesUri = DirectoryUtil.getBlobStoreAllLeasesUri(context); fileStorage.deleteFile(allLeasesUri); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); } catch (UnsupportedFileStorageOperation e) { LogUtil.v( "%s: Failed to release the leases in the android shared storage." @@ -928,12 +1045,12 @@ public class SharedFileManager { TAG); } catch (IOException e) { LogUtil.e(e, "%s: Failed to release the leases in the android shared storage", TAG); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); } } public ListenableFuture<Void> cancelDownloadAndClear() { - return Futures.transformAsync( + return PropagatedFutures.transformAsync( sharedFilesMetadata.getAllFileKeys(), newFileKeyList -> { List<ListenableFuture<Void>> cancelDownloadFutures = new ArrayList<>(); @@ -947,19 +1064,19 @@ public class SharedFileManager { } catch (Exception e) { silentFeedback.send(e, "Failed to cancel all downloads during clear"); } - return Futures.whenAllComplete(cancelDownloadFutures) + return PropagatedFutures.whenAllComplete(cancelDownloadFutures) .callAsync(this::clear, sequentialControlExecutor); }, sequentialControlExecutor); } public ListenableFuture<Void> cancelDownload(NewFileKey newFileKey) { - return Futures.transformAsync( + return PropagatedFutures.transformAsync( sharedFilesMetadata.read(newFileKey), sharedFile -> { if (sharedFile == null) { LogUtil.e("%s: Unable to read sharedFile from shared preferences.", TAG); - return Futures.immediateFailedFuture(new SharedFileMissingException()); + return immediateFailedFuture(new SharedFileMissingException()); } if (sharedFile.getFileStatus() != FileStatus.DOWNLOAD_COMPLETE) { Uri onDeviceUri = @@ -970,9 +1087,19 @@ public class SharedFileManager { newFileKey.getChecksum(), silentFeedback, instanceId, - /* androidShared = */ false); // while downloading androidShared is always false + /* androidShared= */ false); // while downloading androidShared is always false if (onDeviceUri != null) { - fileDownloader.stopDownloading(onDeviceUri); + fileDownloader.stopDownloading(newFileKey.getChecksum(), onDeviceUri); + } + // If the download was in progress, reset it back to subscribed, so it can be properly + // restarted. + if (sharedFile.getFileStatus() == FileStatus.DOWNLOAD_IN_PROGRESS) { + return PropagatedFutures.transformAsync( + sharedFilesMetadata.write( + newFileKey, + sharedFile.toBuilder().setFileStatus(FileStatus.SUBSCRIBED).build()), + unused -> immediateVoidFuture(), + sequentialControlExecutor); } } return immediateVoidFuture(); @@ -983,22 +1110,22 @@ public class SharedFileManager { /** Dumps the current internal state of the SharedFileManager. */ public ListenableFuture<Void> dump(final PrintWriter writer) { writer.println("==== MDD_SHARED_FILES ===="); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( sharedFilesMetadata.getAllFileKeys(), allFileKeys -> { ListenableFuture<Void> writeFilesFuture = immediateVoidFuture(); for (NewFileKey newFileKey : allFileKeys) { writeFilesFuture = - Futures.transformAsync( + PropagatedFutures.transformAsync( writeFilesFuture, voidArg -> - Futures.transformAsync( + PropagatedFutures.transformAsync( sharedFilesMetadata.read(newFileKey), sharedFile -> { if (sharedFile == null) { LogUtil.e( "%s: Unable to read sharedFile from shared preferences.", TAG); - return Futures.immediateVoidFuture(); + return immediateVoidFuture(); } // TODO(b/131166925): MDD dump should not use lite proto toString. writer.format( @@ -1017,14 +1144,14 @@ public class SharedFileManager { newFileKey.getChecksum(), silentFeedback, instanceId, - /* androidShared = */ false); + /* androidShared= */ false); if (serializedUri != null) { writer.format( "Checksum downloaded file: %s\n", FileValidator.computeSha1Digest(fileStorage, serializedUri)); } } - return Futures.immediateVoidFuture(); + return immediateVoidFuture(); }, sequentialControlExecutor), sequentialControlExecutor); diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/SharedFilesMetadata.java b/java/com/google/android/libraries/mobiledatadownload/internal/SharedFilesMetadata.java index c5e6019..df1034e 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/SharedFilesMetadata.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/SharedFilesMetadata.java @@ -18,6 +18,8 @@ package com.google.android.libraries.mobiledatadownload.internal; import android.content.Context; import com.google.android.libraries.mobiledatadownload.SilentFeedback; import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupUtil; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.ListenableFuture; import com.google.mobiledatadownload.internal.MetadataProto.DataFile; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal.AllowedReaders; @@ -126,6 +128,14 @@ public interface SharedFilesMetadata { public ListenableFuture<SharedFile> read(NewFileKey newFileKey); /** + * Returns all known {@link SharedFile}s for the given set of {@link NewFileKey}s + * + * <p>The map will contain a SharedFile entry if it exists. + */ + public ListenableFuture<ImmutableMap<NewFileKey, SharedFile>> readAll( + ImmutableSet<NewFileKey> newFileKeys); + + /** * Map the key "newFileKey" to the value "sharedFile". Returns a future resolving to true if the * operation succeeds, false if it fails. */ diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/SharedPreferencesFileGroupsMetadata.java b/java/com/google/android/libraries/mobiledatadownload/internal/SharedPreferencesFileGroupsMetadata.java index bc407fd..47c0d3d 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/SharedPreferencesFileGroupsMetadata.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/SharedPreferencesFileGroupsMetadata.java @@ -15,22 +15,26 @@ */ package com.google.android.libraries.mobiledatadownload.internal; +import static com.google.common.util.concurrent.Futures.getDone; +import static com.google.common.util.concurrent.Futures.immediateFuture; +import static com.google.common.util.concurrent.Futures.immediateVoidFuture; + import android.content.Context; import android.content.SharedPreferences; -import android.util.Pair; import androidx.annotation.VisibleForTesting; import com.google.android.libraries.mobiledatadownload.SilentFeedback; import com.google.android.libraries.mobiledatadownload.TimeSource; import com.google.android.libraries.mobiledatadownload.annotations.InstanceId; import com.google.android.libraries.mobiledatadownload.internal.annotations.SequentialControlExecutor; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupUtil; import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupsMetadataUtil; import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupsMetadataUtil.GroupKeyDeserializationException; import com.google.android.libraries.mobiledatadownload.internal.util.ProtoLiteUtil; import com.google.android.libraries.mobiledatadownload.internal.util.SharedPreferencesUtil; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Optional; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.errorprone.annotations.CheckReturnValue; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; @@ -81,44 +85,43 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta @Override public ListenableFuture<Void> init() { - return Futures.immediateVoidFuture(); + return immediateVoidFuture(); } @Override public ListenableFuture<@NullableType DataFileGroupInternal> read(GroupKey groupKey) { - String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey, context); + String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey); SharedPreferences prefs = SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); DataFileGroupInternal fileGroup = SharedPreferencesUtil.readProto(prefs, serializedGroupKey, DataFileGroupInternal.parser()); - return Futures.immediateFuture(fileGroup); + return immediateFuture(fileGroup); } @Override public ListenableFuture<Boolean> write(GroupKey groupKey, DataFileGroupInternal fileGroup) { - String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey, context); + String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey); SharedPreferences prefs = SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); - return Futures.immediateFuture( - SharedPreferencesUtil.writeProto(prefs, serializedGroupKey, fileGroup)); + return immediateFuture(SharedPreferencesUtil.writeProto(prefs, serializedGroupKey, fileGroup)); } @Override public ListenableFuture<Boolean> remove(GroupKey groupKey) { - String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey, context); + String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey); SharedPreferences prefs = SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); - return Futures.immediateFuture(SharedPreferencesUtil.removeProto(prefs, serializedGroupKey)); + return immediateFuture(SharedPreferencesUtil.removeProto(prefs, serializedGroupKey)); } @Override public ListenableFuture<@NullableType GroupKeyProperties> readGroupKeyProperties( GroupKey groupKey) { - String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey, context); + String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey); SharedPreferences prefs = SharedPreferencesUtil.getSharedPreferences( @@ -126,18 +129,18 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta GroupKeyProperties groupKeyProperties = SharedPreferencesUtil.readProto(prefs, serializedGroupKey, GroupKeyProperties.parser()); - return Futures.immediateFuture(groupKeyProperties); + return immediateFuture(groupKeyProperties); } @Override public ListenableFuture<Boolean> writeGroupKeyProperties( GroupKey groupKey, GroupKeyProperties groupKeyProperties) { - String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey, context); + String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey); SharedPreferences prefs = SharedPreferencesUtil.getSharedPreferences( context, MDD_FILE_GROUP_KEY_PROPERTIES, instanceId); - return Futures.immediateFuture( + return immediateFuture( SharedPreferencesUtil.writeProto(prefs, serializedGroupKey, groupKeyProperties)); } @@ -169,12 +172,12 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta if (editor != null) { editor.commit(); } - return Futures.immediateFuture(groupKeyList); + return immediateFuture(groupKeyList); } @Override - public ListenableFuture<List<Pair<GroupKey, DataFileGroupInternal>>> getAllFreshGroups() { - return Futures.transformAsync( + public ListenableFuture<List<GroupKeyAndGroup>> getAllFreshGroups() { + return PropagatedFutures.transformAsync( getAllGroupKeys(), groupKeyList -> { List<ListenableFuture<@NullableType DataFileGroupInternal>> groupReadFutures = @@ -182,19 +185,19 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta for (GroupKey key : groupKeyList) { groupReadFutures.add(read(key)); } - return Futures.whenAllComplete(groupReadFutures) + return PropagatedFutures.whenAllComplete(groupReadFutures) .callAsync( () -> { - List<Pair<GroupKey, DataFileGroupInternal>> retrievedGroups = new ArrayList<>(); + List<GroupKeyAndGroup> retrievedGroups = new ArrayList<>(); for (int i = 0; i < groupKeyList.size(); i++) { GroupKey key = groupKeyList.get(i); - DataFileGroupInternal group = Futures.getDone(groupReadFutures.get(i)); + DataFileGroupInternal group = getDone(groupReadFutures.get(i)); if (group == null) { continue; } - retrievedGroups.add(Pair.create(key, group)); + retrievedGroups.add(GroupKeyAndGroup.create(key, group)); } - return Futures.immediateFuture(retrievedGroups); + return immediateFuture(retrievedGroups); }, sequentialControlExecutor); }, @@ -210,12 +213,12 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta LogUtil.d("%s: Removing group %s %s", TAG, key.getGroupName(), key.getOwnerPackage()); SharedPreferencesUtil.removeProto(editor, key); } - return Futures.immediateFuture(editor.commit()); + return immediateFuture(editor.commit()); } @Override public ListenableFuture<List<DataFileGroupInternal>> getAllStaleGroups() { - return Futures.immediateFuture( + return immediateFuture( FileGroupsMetadataUtil.getAllStaleGroups( FileGroupsMetadataUtil.getGarbageCollectorFile(context, instanceId))); } @@ -243,7 +246,7 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta outputStream = new FileOutputStream(garbageCollectorFile, /* append */ true); } catch (FileNotFoundException e) { LogUtil.e("File %s not found while writing.", garbageCollectorFile.getAbsolutePath()); - return Futures.immediateFuture(false); + return immediateFuture(false); } try { @@ -255,9 +258,9 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta outputStream.close(); } catch (IOException e) { LogUtil.e("IOException occurred while writing file groups."); - return Futures.immediateFuture(false); + return immediateFuture(false); } - return Futures.immediateFuture(true); + return immediateFuture(true); } @VisibleForTesting @@ -269,7 +272,7 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta @Override public ListenableFuture<Void> removeAllStaleGroups() { getGarbageCollectorFile().delete(); - return Futures.immediateVoidFuture(); + return immediateVoidFuture(); } @Override diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/SharedPreferencesSharedFilesMetadata.java b/java/com/google/android/libraries/mobiledatadownload/internal/SharedPreferencesSharedFilesMetadata.java index 662ac5b..851225b 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/SharedPreferencesSharedFilesMetadata.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/SharedPreferencesSharedFilesMetadata.java @@ -17,10 +17,13 @@ package com.google.android.libraries.mobiledatadownload.internal; import static com.google.android.libraries.mobiledatadownload.internal.MddConstants.SPLIT_CHAR; import static com.google.android.libraries.mobiledatadownload.internal.util.SharedFilesMetadataUtil.MDD_SHARED_FILES; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import android.content.Context; import android.content.SharedPreferences; + import androidx.annotation.VisibleForTesting; + import com.google.android.libraries.mobiledatadownload.Flags; import com.google.android.libraries.mobiledatadownload.SilentFeedback; import com.google.android.libraries.mobiledatadownload.annotations.InstanceId; @@ -29,15 +32,20 @@ import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.android.libraries.mobiledatadownload.internal.util.SharedFilesMetadataUtil; import com.google.android.libraries.mobiledatadownload.internal.util.SharedFilesMetadataUtil.FileKeyDeserializationException; import com.google.android.libraries.mobiledatadownload.internal.util.SharedPreferencesUtil; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Optional; import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.errorprone.annotations.CheckReturnValue; import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey; import com.google.mobiledatadownload.internal.MetadataProto.SharedFile; + import java.util.ArrayList; import java.util.List; + import javax.inject.Inject; /** @@ -49,281 +57,305 @@ import javax.inject.Inject; @CheckReturnValue public final class SharedPreferencesSharedFilesMetadata implements SharedFilesMetadata { - private static final String TAG = "SharedFilesMetadata"; - - @VisibleForTesting static final String PREFS_KEY_NEXT_FILE_NAME_OLD = "next_file_name"; - @VisibleForTesting static final String PREFS_KEY_NEXT_FILE_NAME = "next_file_name_v2"; - - private final Context context; - private final SilentFeedback silentFeedback; - private final Optional<String> instanceId; - private final Flags flags; - - @Inject - public SharedPreferencesSharedFilesMetadata( - @ApplicationContext Context context, - SilentFeedback silentFeedback, - @InstanceId Optional<String> instanceId, - Flags flags) { - this.context = context; - this.silentFeedback = silentFeedback; - this.instanceId = instanceId; - this.flags = flags; - } - - @Override - public ListenableFuture<Boolean> init() { - // Migrate to the new file key. - if (!Migrations.isMigratedToNewFileKey(context)) { - LogUtil.d("%s Device isn't migrated to new file key, clear and set migration.", TAG); - Migrations.setMigratedToNewFileKey(context, true); - Migrations.setCurrentVersion(context, FileKeyVersion.getVersion(flags.fileKeyVersion())); - return Futures.immediateFuture(false); + private static final String TAG = "SharedFilesMetadata"; + + @VisibleForTesting + static final String PREFS_KEY_NEXT_FILE_NAME_OLD = "next_file_name"; + @VisibleForTesting + static final String PREFS_KEY_NEXT_FILE_NAME = "next_file_name_v2"; + + private final Context context; + private final SilentFeedback silentFeedback; + private final Optional<String> instanceId; + private final Flags flags; + + @Inject + public SharedPreferencesSharedFilesMetadata( + @ApplicationContext Context context, + SilentFeedback silentFeedback, + @InstanceId Optional<String> instanceId, + Flags flags) { + this.context = context; + this.silentFeedback = silentFeedback; + this.instanceId = instanceId; + this.flags = flags; } - return Futures.immediateFuture(upgradeToNewVersion()); - } - - /** - * Sequentially upgrade FileKey version to FeatureFlags.fileKeyVersion - * - * @return false if any upgrade fails which will result in clearing of all meta data, true on - * successful upgrade. - */ - private boolean upgradeToNewVersion() { - final FileKeyVersion targetVersion = FileKeyVersion.getVersion(flags.fileKeyVersion()); - final FileKeyVersion currentVersion = Migrations.getCurrentVersion(context, silentFeedback); - - if (targetVersion.value == currentVersion.value) { - return true; + + @Override + public ListenableFuture<Boolean> init() { + // Migrate to the new file key. + if (!Migrations.isMigratedToNewFileKey(context)) { + LogUtil.d("%s Device isn't migrated to new file key, clear and set migration.", TAG); + Migrations.setMigratedToNewFileKey(context, true); + Migrations.setCurrentVersion(context, + FileKeyVersion.getVersion(flags.fileKeyVersion())); + return Futures.immediateFuture(false); + } + return Futures.immediateFuture(upgradeToNewVersion()); } - if (targetVersion.value < currentVersion.value) { - // We don't support downgrading file key version. Clear everything. - LogUtil.e( - "%s Cannot migrate back from value %s to %s. Clear everything!", - TAG, currentVersion, targetVersion); - silentFeedback.send( - new Exception( - "Downgraded file key from " + currentVersion + " to " + targetVersion + "."), - "FileKey migrations unexpected downgrade."); - Migrations.setCurrentVersion(context, targetVersion); - return false; + /** + * Sequentially upgrade FileKey version to FeatureFlags.fileKeyVersion + * + * @return false if any upgrade fails which will result in clearing of all meta data, true on + * successful upgrade. + */ + private boolean upgradeToNewVersion() { + final FileKeyVersion targetVersion = FileKeyVersion.getVersion(flags.fileKeyVersion()); + final FileKeyVersion currentVersion = Migrations.getCurrentVersion(context, silentFeedback); + + if (targetVersion.value == currentVersion.value) { + return true; + } + + if (targetVersion.value < currentVersion.value) { + // We don't support downgrading file key version. Clear everything. + LogUtil.e( + "%s Cannot migrate back from value %s to %s. Clear everything!", + TAG, currentVersion, targetVersion); + silentFeedback.send( + new Exception( + "Downgraded file key from " + currentVersion + " to " + targetVersion + + "."), + "FileKey migrations unexpected downgrade."); + Migrations.setCurrentVersion(context, targetVersion); + return false; + } + + // Migrate one version at a time one by one + try { + for (int nextVersion = currentVersion.value + 1; + nextVersion <= targetVersion.value; + nextVersion++) { + if (upgradeTo(FileKeyVersion.getVersion(nextVersion))) { + Migrations.setCurrentVersion(context, FileKeyVersion.getVersion(nextVersion)); + } else { + // If migration to next version fail, we will clear all data and set the + // currentVersion + // to targetVersion (phFileKeyVersion) + return false; + } + } + } finally { + if (Migrations.getCurrentVersion(context, silentFeedback).value + != targetVersion.value) { + if (!Migrations.setCurrentVersion(context, targetVersion)) { + LogUtil.e( + "Failed to commit migration version to disk. Fail to set target " + + "version to " + + targetVersion + + "."); + silentFeedback.send( + new Exception("Fail to set target version " + targetVersion + "."), + "Failed to commit migration version to disk."); + } + } + } + + return true; } - // Migrate one version at a time one by one - try { - for (int nextVersion = currentVersion.value + 1; - nextVersion <= targetVersion.value; - nextVersion++) { - if (upgradeTo(FileKeyVersion.getVersion(nextVersion))) { - Migrations.setCurrentVersion(context, FileKeyVersion.getVersion(nextVersion)); - } else { - // If migration to next version fail, we will clear all data and set the currentVersion - // to targetVersion (phFileKeyVersion) - return false; + private boolean upgradeTo(FileKeyVersion targetVersion) { + switch (targetVersion) { + case ADD_DOWNLOAD_TRANSFORM: + return migrateToAddDownloadTransform(); + case USE_CHECKSUM_ONLY: + return migrateToDedupOnChecksumOnly(); + default: + throw new UnsupportedOperationException( + "Upgrade to version " + targetVersion.name() + "not supported!"); } - } - } finally { - if (Migrations.getCurrentVersion(context, silentFeedback).value != targetVersion.value) { - if (!Migrations.setCurrentVersion(context, targetVersion)) { - LogUtil.e( - "Failed to commit migration version to disk. Fail to set target version to " - + targetVersion - + "."); - silentFeedback.send( - new Exception("Fail to set target version " + targetVersion + "."), - "Failed to commit migration version to disk."); + } + + /** A one off method that is called when we migrate key to add download transform. */ + private boolean migrateToAddDownloadTransform() { + LogUtil.d("%s: Starting migration to add download transform", TAG); + SharedPreferences prefs = + SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); + SharedPreferences.Editor editor = prefs.edit(); + for (String serializedFileKey : prefs.getAll().keySet()) { + + // Remove the data that we are unable to read or parse. + NewFileKey newFileKey; + try { + newFileKey = + SharedFilesMetadataUtil.deserializeNewFileKey( + serializedFileKey, context, silentFeedback); + } catch (FileKeyDeserializationException e) { + LogUtil.e( + "%s Failed to deserialize file key %s, remove and continue.", TAG, + serializedFileKey); + silentFeedback.send(e, "Failed to deserialize file key, remove and continue."); + editor.remove(serializedFileKey); + continue; + } + SharedFile sharedFile = + SharedPreferencesUtil.readProto(prefs, serializedFileKey, SharedFile.parser()); + if (sharedFile == null) { + LogUtil.e("%s: Unable to read sharedFile from shared preferences.", TAG); + editor.remove(serializedFileKey); + continue; + } + + // Remove the old key and write the new one. + SharedPreferencesUtil.removeProto(editor, serializedFileKey); + SharedPreferencesUtil.writeProto( + editor, + SharedFilesMetadataUtil.serializeNewFileKeyWithDownloadTransform(newFileKey), + sharedFile); } - } + + if (!editor.commit()) { + LogUtil.e("Failed to commit migration metadata to disk"); + silentFeedback.send( + new Exception("Migrate to DownloadTransform failed."), + "Failed to commit migration metadata to disk."); + return false; + } + + return true; } - return true; - } - - private boolean upgradeTo(FileKeyVersion targetVersion) { - switch (targetVersion) { - case ADD_DOWNLOAD_TRANSFORM: - return migrateToAddDownloadTransform(); - case USE_CHECKSUM_ONLY: - return migrateToDedupOnChecksumOnly(); - default: - throw new UnsupportedOperationException( - "Upgrade to version " + targetVersion.name() + "not supported!"); + /** A one off method that is called when we migrate key to contain checksum and + * allowedReaders. */ + private boolean migrateToDedupOnChecksumOnly() { + LogUtil.d("%s: Starting migration to dedup on checksum only", TAG); + SharedPreferences prefs = + SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); + SharedPreferences.Editor editor = prefs.edit(); + for (String serializedFileKey : prefs.getAll().keySet()) { + + // Remove the data that we are unable to read or parse. + NewFileKey newFileKey; + try { + newFileKey = + SharedFilesMetadataUtil.deserializeNewFileKey( + serializedFileKey, context, silentFeedback); + } catch (FileKeyDeserializationException e) { + LogUtil.e( + "%s Failed to deserialize file key %s, remove and continue.", TAG, + serializedFileKey); + silentFeedback.send(e, "Failed to deserialize file key, remove and continue."); + editor.remove(serializedFileKey); + continue; + } + + SharedFile sharedFile = + SharedPreferencesUtil.readProto(prefs, serializedFileKey, SharedFile.parser()); + if (sharedFile == null) { + LogUtil.e("%s: Unable to read sharedFile from shared preferences.", TAG); + editor.remove(serializedFileKey); + continue; + } + + // Remove the old key and write the new one. + SharedPreferencesUtil.removeProto(editor, serializedFileKey); + SharedPreferencesUtil.writeProto( + editor, + SharedFilesMetadataUtil.serializeNewFileKeyWithChecksumOnly(newFileKey), + sharedFile); + } + + if (!editor.commit()) { + LogUtil.e("Failed to commit migration metadata to disk"); + silentFeedback.send( + new Exception("Migrate to ChecksumOnly failed."), + "Failed to commit migration metadata to disk."); + return false; + } + + return true; } - } - - /** A one off method that is called when we migrate key to add download transform. */ - private boolean migrateToAddDownloadTransform() { - LogUtil.d("%s: Starting migration to add download transform", TAG); - SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); - SharedPreferences.Editor editor = prefs.edit(); - for (String serializedFileKey : prefs.getAll().keySet()) { - - // Remove the data that we are unable to read or parse. - NewFileKey newFileKey; - try { - newFileKey = - SharedFilesMetadataUtil.deserializeNewFileKey( - serializedFileKey, context, silentFeedback); - } catch (FileKeyDeserializationException e) { - LogUtil.e( - "%s Failed to deserialize file key %s, remove and continue.", TAG, serializedFileKey); - silentFeedback.send(e, "Failed to deserialize file key, remove and continue."); - editor.remove(serializedFileKey); - continue; - } - SharedFile sharedFile = - SharedPreferencesUtil.readProto(prefs, serializedFileKey, SharedFile.parser()); - if (sharedFile == null) { - LogUtil.e("%s: Unable to read sharedFile from shared preferences.", TAG); - editor.remove(serializedFileKey); - continue; - } - - // Remove the old key and write the new one. - SharedPreferencesUtil.removeProto(editor, serializedFileKey); - SharedPreferencesUtil.writeProto( - editor, - SharedFilesMetadataUtil.serializeNewFileKeyWithDownloadTransform(newFileKey), - sharedFile); + + @SuppressWarnings("nullness") + @Override + public ListenableFuture<SharedFile> read(NewFileKey newFileKey) { + return PropagatedFutures.transform( + readAll(ImmutableSet.of(newFileKey)), + sharedFiles -> sharedFiles.get(newFileKey), + directExecutor()); } - if (!editor.commit()) { - LogUtil.e("Failed to commit migration metadata to disk"); - silentFeedback.send( - new Exception("Migrate to DownloadTransform failed."), - "Failed to commit migration metadata to disk."); - return false; + @Override + public ListenableFuture<ImmutableMap<NewFileKey, SharedFile>> readAll( + ImmutableSet<NewFileKey> newFileKeys) { + SharedPreferences prefs = + SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); + ImmutableMap.Builder<NewFileKey, SharedFile> sharedFileMapBuilder = ImmutableMap.builder(); + for (NewFileKey newFileKey : newFileKeys) { + String serializedFileKey = + SharedFilesMetadataUtil.getSerializedFileKey(newFileKey, context, + silentFeedback); + SharedFile sharedFile = + SharedPreferencesUtil.readProto(prefs, serializedFileKey, SharedFile.parser()); + if (sharedFile != null) { + sharedFileMapBuilder.put(newFileKey, sharedFile); + } + } + return Futures.immediateFuture(sharedFileMapBuilder.build()); } - return true; - } - - /** A one off method that is called when we migrate key to contain checksum and allowedReaders. */ - private boolean migrateToDedupOnChecksumOnly() { - LogUtil.d("%s: Starting migration to dedup on checksum only", TAG); - SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); - SharedPreferences.Editor editor = prefs.edit(); - for (String serializedFileKey : prefs.getAll().keySet()) { - - // Remove the data that we are unable to read or parse. - NewFileKey newFileKey; - try { - newFileKey = - SharedFilesMetadataUtil.deserializeNewFileKey( - serializedFileKey, context, silentFeedback); - } catch (FileKeyDeserializationException e) { - LogUtil.e( - "%s Failed to deserialize file key %s, remove and continue.", TAG, serializedFileKey); - silentFeedback.send(e, "Failed to deserialize file key, remove and continue."); - editor.remove(serializedFileKey); - continue; - } - - SharedFile sharedFile = - SharedPreferencesUtil.readProto(prefs, serializedFileKey, SharedFile.parser()); - if (sharedFile == null) { - LogUtil.e("%s: Unable to read sharedFile from shared preferences.", TAG); - editor.remove(serializedFileKey); - continue; - } - - // Remove the old key and write the new one. - SharedPreferencesUtil.removeProto(editor, serializedFileKey); - SharedPreferencesUtil.writeProto( - editor, - SharedFilesMetadataUtil.serializeNewFileKeyWithChecksumOnly(newFileKey), - sharedFile); + @Override + public ListenableFuture<Boolean> write(NewFileKey newFileKey, SharedFile sharedFile) { + String serializedFileKey = + SharedFilesMetadataUtil.getSerializedFileKey(newFileKey, context, silentFeedback); + + SharedPreferences prefs = + SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); + return Futures.immediateFuture( + SharedPreferencesUtil.writeProto(prefs, serializedFileKey, sharedFile)); } - if (!editor.commit()) { - LogUtil.e("Failed to commit migration metadata to disk"); - silentFeedback.send( - new Exception("Migrate to ChecksumOnly failed."), - "Failed to commit migration metadata to disk."); - return false; + @Override + public ListenableFuture<Boolean> remove(NewFileKey newFileKey) { + String serializedFileKey = + SharedFilesMetadataUtil.getSerializedFileKey(newFileKey, context, silentFeedback); + + SharedPreferences prefs = + SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); + return Futures.immediateFuture(SharedPreferencesUtil.removeProto(prefs, serializedFileKey)); } - return true; - } - - @SuppressWarnings("nullness") - @Override - public ListenableFuture<SharedFile> read(NewFileKey newFileKey) { - String serializedFileKey = - SharedFilesMetadataUtil.getSerializedFileKey(newFileKey, context, silentFeedback); - - SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); - SharedFile sharedFile = - SharedPreferencesUtil.readProto(prefs, serializedFileKey, SharedFile.parser()); - - return Futures.immediateFuture(sharedFile); - } - - @Override - public ListenableFuture<Boolean> write(NewFileKey newFileKey, SharedFile sharedFile) { - String serializedFileKey = - SharedFilesMetadataUtil.getSerializedFileKey(newFileKey, context, silentFeedback); - - SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); - return Futures.immediateFuture( - SharedPreferencesUtil.writeProto(prefs, serializedFileKey, sharedFile)); - } - - @Override - public ListenableFuture<Boolean> remove(NewFileKey newFileKey) { - String serializedFileKey = - SharedFilesMetadataUtil.getSerializedFileKey(newFileKey, context, silentFeedback); - - SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); - return Futures.immediateFuture(SharedPreferencesUtil.removeProto(prefs, serializedFileKey)); - } - - @Override - public ListenableFuture<List<NewFileKey>> getAllFileKeys() { - List<NewFileKey> newFileKeyList = new ArrayList<>(); - SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); - SharedPreferences.Editor editor = null; - for (String serializedFileKey : prefs.getAll().keySet()) { - try { - NewFileKey newFileKey = - SharedFilesMetadataUtil.deserializeNewFileKey( - serializedFileKey, context, silentFeedback); - newFileKeyList.add(newFileKey); - } catch (FileKeyDeserializationException e) { - LogUtil.e(e, "Failed to deserialize newFileKey:" + serializedFileKey); - silentFeedback.send( - e, - "Failed to deserialize newFileKey, unexpected key size: %d", - Splitter.on(SPLIT_CHAR).splitToList(serializedFileKey).size()); - // TODO(b/128850000): Refactor this code to a single corruption handling task during - // maintenance. - // Remove the corrupted file metadata and the related FileGroup metadata will be deleted - // in next maintenance task. - if (editor == null) { - editor = prefs.edit(); + @Override + public ListenableFuture<List<NewFileKey>> getAllFileKeys() { + List<NewFileKey> newFileKeyList = new ArrayList<>(); + SharedPreferences prefs = + SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); + SharedPreferences.Editor editor = null; + for (String serializedFileKey : prefs.getAll().keySet()) { + try { + NewFileKey newFileKey = + SharedFilesMetadataUtil.deserializeNewFileKey( + serializedFileKey, context, silentFeedback); + newFileKeyList.add(newFileKey); + } catch (FileKeyDeserializationException e) { + LogUtil.e(e, "Failed to deserialize newFileKey:" + serializedFileKey); + silentFeedback.send( + e, + "Failed to deserialize newFileKey, unexpected key size: %d", + Splitter.on(SPLIT_CHAR).splitToList(serializedFileKey).size()); + // TODO(b/128850000): Refactor this code to a single corruption handling task during + // maintenance. + // Remove the corrupted file metadata and the related FileGroup metadata will be deleted + // in next maintenance task. + if (editor == null) { + editor = prefs.edit(); + } + editor.remove(serializedFileKey); + continue; + } + } + if (editor != null) { + editor.commit(); } - editor.remove(serializedFileKey); - continue; - } + return Futures.immediateFuture(newFileKeyList); } - if (editor != null) { - editor.commit(); + + @Override + public ListenableFuture<Void> clear() { + SharedPreferences prefs = + SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); + prefs.edit().clear().commit(); + return Futures.immediateFuture(null); } - return Futures.immediateFuture(newFileKeyList); - } - - @Override - public ListenableFuture<Void> clear() { - SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); - prefs.edit().clear().commit(); - return Futures.immediateFuture(null); - } } diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/annotations/BUILD b/java/com/google/android/libraries/mobiledatadownload/internal/annotations/BUILD index dc959e6..a6b734f 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/annotations/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/internal/annotations/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/collect/BUILD b/java/com/google/android/libraries/mobiledatadownload/internal/collect/BUILD new file mode 100644 index 0000000..c381862 --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/internal/collect/BUILD @@ -0,0 +1,33 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +load("@build_bazel_rules_android//android:rules.bzl", "android_library") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = [ + "//visibility:public", + ], + licenses = ["notice"], +) + +android_library( + name = "collect", + srcs = glob(["*.java"]), + deps = [ + "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "@com_google_auto_value", + "@com_google_code_findbugs_jsr305", + "@com_google_errorprone_error_prone_annotations", + ], +) diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/collect/GroupKeyAndGroup.java b/java/com/google/android/libraries/mobiledatadownload/internal/collect/GroupKeyAndGroup.java new file mode 100644 index 0000000..c84a8d6 --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/internal/collect/GroupKeyAndGroup.java @@ -0,0 +1,34 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.internal.collect; + +import com.google.auto.value.AutoValue; +import com.google.errorprone.annotations.Immutable; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; +import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; + +/** Container for associated {@link GroupKey} and {@link DataFileGroupInternal}. */ +@AutoValue +@Immutable +public abstract class GroupKeyAndGroup { + public static GroupKeyAndGroup create(GroupKey groupKey, DataFileGroupInternal dataFileGroup) { + return new AutoValue_GroupKeyAndGroup(groupKey, dataFileGroup); + } + + public abstract GroupKey groupKey(); + + public abstract DataFileGroupInternal dataFileGroup(); +} diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/collect/GroupPair.java b/java/com/google/android/libraries/mobiledatadownload/internal/collect/GroupPair.java new file mode 100644 index 0000000..3a76887 --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/internal/collect/GroupPair.java @@ -0,0 +1,38 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.internal.collect; + +import com.google.auto.value.AutoValue; +import com.google.errorprone.annotations.Immutable; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; +import javax.annotation.Nullable; + +/** Container for associated downloaded and pending versions of the same group. */ +@AutoValue +@Immutable +public abstract class GroupPair { + public static GroupPair create( + @Nullable DataFileGroupInternal pendingGroup, + @Nullable DataFileGroupInternal downloadedGroup) { + return new AutoValue_GroupPair(pendingGroup, downloadedGroup); + } + + @Nullable + public abstract DataFileGroupInternal pendingGroup(); + + @Nullable + public abstract DataFileGroupInternal downloadedGroup(); +} diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/dagger/BUILD b/java/com/google/android/libraries/mobiledatadownload/internal/dagger/BUILD index 065c222..7255a39 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/dagger/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/internal/dagger/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -60,17 +61,20 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload:TimeSource", "//java/com/google/android/libraries/mobiledatadownload/annotations", "//java/com/google/android/libraries/mobiledatadownload/file", + "//java/com/google/android/libraries/mobiledatadownload/internal:AndroidTimeSource", "//java/com/google/android/libraries/mobiledatadownload/internal:ApplicationContext", "//java/com/google/android/libraries/mobiledatadownload/internal:FileGroupsMetadata", "//java/com/google/android/libraries/mobiledatadownload/internal:SharedFilesMetadata", "//java/com/google/android/libraries/mobiledatadownload/internal:SharedPreferencesFileGroupsMetadata", "//java/com/google/android/libraries/mobiledatadownload/internal:SharedPreferencesSharedFilesMetadata", + "//java/com/google/android/libraries/mobiledatadownload/internal/annotations", "//java/com/google/android/libraries/mobiledatadownload/internal/annotations:SequentialControlExecutor", "//java/com/google/android/libraries/mobiledatadownload/internal/experimentation:DownloadStageManager", "//java/com/google/android/libraries/mobiledatadownload/internal/experimentation:NoOpDownloadStageManager", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LoggingStateStore", - "//java/com/google/android/libraries/mobiledatadownload/internal/logging:NoOpLoggingState", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:SharedPreferencesLoggingState", + "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//java/com/google/android/libraries/mobiledatadownload/internal/util:FuturesUtil", "//java/com/google/android/libraries/mobiledatadownload/monitor:DownloadProgressMonitor", "//java/com/google/android/libraries/mobiledatadownload/monitor:NetworkUsageMonitor", @@ -90,6 +94,7 @@ android_library( ":DownloaderModule", ":ExecutorsModule", ":MainMddLibModule", + "//java/com/google/android/libraries/mobiledatadownload:TimeSource", "//java/com/google/android/libraries/mobiledatadownload/internal:MobileDataDownloadManager", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LoggingStateStore", diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/dagger/MainMddLibModule.java b/java/com/google/android/libraries/mobiledatadownload/internal/dagger/MainMddLibModule.java index 18c23f0..39957f0 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/dagger/MainMddLibModule.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/dagger/MainMddLibModule.java @@ -23,6 +23,7 @@ import com.google.android.libraries.mobiledatadownload.SilentFeedback; import com.google.android.libraries.mobiledatadownload.TimeSource; import com.google.android.libraries.mobiledatadownload.annotations.InstanceId; import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; +import com.google.android.libraries.mobiledatadownload.internal.AndroidTimeSource; import com.google.android.libraries.mobiledatadownload.internal.ApplicationContext; import com.google.android.libraries.mobiledatadownload.internal.FileGroupsMetadata; import com.google.android.libraries.mobiledatadownload.internal.SharedFilesMetadata; @@ -33,13 +34,14 @@ import com.google.android.libraries.mobiledatadownload.internal.experimentation. import com.google.android.libraries.mobiledatadownload.internal.experimentation.NoOpDownloadStageManager; import com.google.android.libraries.mobiledatadownload.internal.logging.EventLogger; import com.google.android.libraries.mobiledatadownload.internal.logging.LoggingStateStore; -import com.google.android.libraries.mobiledatadownload.internal.logging.NoOpLoggingState; +import com.google.android.libraries.mobiledatadownload.internal.logging.SharedPreferencesLoggingState; import com.google.android.libraries.mobiledatadownload.internal.util.FuturesUtil; import com.google.android.libraries.mobiledatadownload.monitor.DownloadProgressMonitor; import com.google.android.libraries.mobiledatadownload.monitor.NetworkUsageMonitor; import com.google.common.base.Optional; import dagger.Module; import dagger.Provides; +import java.security.SecureRandom; import java.util.concurrent.Executor; import javax.inject.Singleton; @@ -49,17 +51,25 @@ public class MainMddLibModule { /** The version of MDD library. Same as mdi_download module version. */ // TODO(b/122271766): Figure out how to update this automatically. // LINT.IfChange - public static final int MDD_LIB_VERSION = 422883838; + public static final int MDD_LIB_VERSION = 516938429; // LINT.ThenChange(<internal>) private final SynchronousFileStorage fileStorage; + private final NetworkUsageMonitor networkUsageMonitor; + private final EventLogger eventLogger; + private final Optional<DownloadProgressMonitor> downloadProgressMonitorOptional; + private final Optional<SilentFeedback> silentFeedbackOptional; + private final Optional<String> instanceId; + private final Optional<AccountSource> accountSourceOptional; + private final Flags flags; + private final Optional<ExperimentationConfig> experimentationConfigOptional; public MainMddLibModule( @@ -99,12 +109,14 @@ public class MainMddLibModule { @Provides @Singleton + @SuppressWarnings("Framework.StaticProvides") EventLogger provideEventLogger() { return eventLogger; } @Provides @Singleton + @SuppressWarnings("Framework.StaticProvides") SilentFeedback providesSilentFeedback() { if (this.silentFeedbackOptional.isPresent()) { return this.silentFeedbackOptional.get(); @@ -117,6 +129,7 @@ public class MainMddLibModule { @Provides @Singleton + @SuppressWarnings("Framework.StaticProvides") Optional<AccountSource> provideAccountSourceOptional(@ApplicationContext Context context) { return this.accountSourceOptional; } @@ -124,24 +137,27 @@ public class MainMddLibModule { @Provides @Singleton static TimeSource provideTimeSource() { - return System::currentTimeMillis; + return new AndroidTimeSource(); } @Provides @Singleton @InstanceId + @SuppressWarnings("Framework.StaticProvides") Optional<String> provideInstanceId() { return this.instanceId; } @Provides @Singleton + @SuppressWarnings("Framework.StaticProvides") NetworkUsageMonitor provideNetworkUsageMonitor() { return this.networkUsageMonitor; } @Provides @Singleton + @SuppressWarnings("Framework.StaticProvides") // TODO: We don't need to have @Singleton here and few other places in this class // since it comes from the this instance. We should remove this since it could increase APK size. Optional<DownloadProgressMonitor> provideDownloadProgressMonitor() { @@ -150,17 +166,20 @@ public class MainMddLibModule { @Provides @Singleton + @SuppressWarnings("Framework.StaticProvides") SynchronousFileStorage provideSynchronousFileStorage() { return this.fileStorage; } @Provides @Singleton + @SuppressWarnings("Framework.StaticProvides") Flags provideFlags() { return this.flags; } @Provides + @SuppressWarnings("Framework.StaticProvides") Optional<ExperimentationConfig> provideExperimentationConfigOptional() { return this.experimentationConfigOptional; } @@ -173,12 +192,18 @@ public class MainMddLibModule { @Provides @Singleton - static LoggingStateStore provideLoggingStateStore() { - return new NoOpLoggingState(); + static LoggingStateStore provideLoggingStateStore( + @ApplicationContext Context context, + @InstanceId Optional<String> instanceId, + TimeSource timeSource, + @SequentialControlExecutor Executor sequentialExecutor) { + return SharedPreferencesLoggingState.createFromContext( + context, instanceId, timeSource, sequentialExecutor, new SecureRandom()); } @Provides - static DownloadStageManager provideDownloadStageManager( + @SuppressWarnings("Framework.StaticProvides") + DownloadStageManager provideDownloadStageManager( FileGroupsMetadata fileGroupsMetadata, Optional<ExperimentationConfig> experimentationConfigOptional, @SequentialControlExecutor Executor executor, diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/dagger/StandaloneComponent.java b/java/com/google/android/libraries/mobiledatadownload/internal/dagger/StandaloneComponent.java index 48367a4..b4cae41 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/dagger/StandaloneComponent.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/dagger/StandaloneComponent.java @@ -15,6 +15,7 @@ */ package com.google.android.libraries.mobiledatadownload.internal.dagger; +import com.google.android.libraries.mobiledatadownload.TimeSource; import com.google.android.libraries.mobiledatadownload.internal.MobileDataDownloadManager; import com.google.android.libraries.mobiledatadownload.internal.logging.EventLogger; import com.google.android.libraries.mobiledatadownload.internal.logging.LoggingStateStore; @@ -38,4 +39,6 @@ public abstract class StandaloneComponent { // TODO(b/214632773): remove this when event logger can be constructed internally public abstract LoggingStateStore getLoggingStateStore(); + + public abstract TimeSource getTimeSource(); } diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/downloader/BUILD b/java/com/google/android/libraries/mobiledatadownload/internal/downloader/BUILD index 5d239c1..4288856 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/downloader/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/internal/downloader/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -34,8 +35,11 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LoggingStateStore", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "//java/com/google/android/libraries/mobiledatadownload/internal/util:DownloadFutureMap", + "//java/com/google/android/libraries/mobiledatadownload/internal/util:FileGroupUtil", "//java/com/google/android/libraries/mobiledatadownload/monitor:DownloadProgressMonitor", "//java/com/google/android/libraries/mobiledatadownload/monitor:NetworkUsageMonitor", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "@androidx_annotation_annotation", "@com_google_code_findbugs_jsr305", "@com_google_dagger", @@ -85,6 +89,8 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//java/com/google/android/libraries/mobiledatadownload/internal/util:FileGroupUtil", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "@androidx_annotation_annotation", "@com_google_guava_guava", ], @@ -109,6 +115,8 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//java/com/google/android/libraries/mobiledatadownload/internal/util:DirectoryUtil", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "@com_google_guava_guava", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/downloader/DeltaFileDownloaderCallbackImpl.java b/java/com/google/android/libraries/mobiledatadownload/internal/downloader/DeltaFileDownloaderCallbackImpl.java index a84397a..345616d 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/downloader/DeltaFileDownloaderCallbackImpl.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/downloader/DeltaFileDownloaderCallbackImpl.java @@ -44,6 +44,7 @@ import com.google.mobiledatadownload.internal.MetadataProto.DeltaFile; import com.google.mobiledatadownload.internal.MetadataProto.FileStatus; import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; import java.io.IOException; import java.util.concurrent.Executor; @@ -210,7 +211,7 @@ public final class DeltaFileDownloaderCallbackImpl implements DownloaderCallback baseFileKey.getChecksum(), silentFeedback, instanceId, - /* androidShared = */ false); + /* androidShared= */ false); } if (baseFileUri == null) { @@ -237,7 +238,14 @@ public final class DeltaFileDownloaderCallbackImpl implements DownloaderCallback .setCause(e) .build()); } - Void fileGroupStats = null; + DataDownloadFileGroupStats fileGroupStats = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(groupKey.getGroupName()) + .setFileGroupVersionNumber(fileGroupVersionNumber) + .setOwnerPackage(groupKey.getOwnerPackage()) + .setBuildId(buildId) + .setVariantId(variantId) + .build(); eventLogger.logMddNetworkSavings( fileGroupStats, 0, diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/downloader/DownloaderCallbackImpl.java b/java/com/google/android/libraries/mobiledatadownload/internal/downloader/DownloaderCallbackImpl.java index 897261d..bd784b3 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/downloader/DownloaderCallbackImpl.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/downloader/DownloaderCallbackImpl.java @@ -40,6 +40,8 @@ import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentF import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.io.ByteStreams; import com.google.common.util.concurrent.ListenableFuture; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; import com.google.mobiledatadownload.internal.MetadataProto.DataFile; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal.AllowedReaders; import com.google.mobiledatadownload.internal.MetadataProto.FileStatus; @@ -262,14 +264,21 @@ public class DownloaderCallbackImpl implements DownloaderCallback { long fullFileSize = fileStorage.fileSize(target); long downloadedFileSize = fileStorage.fileSize(source); if (fullFileSize > downloadedFileSize) { - Void fileGroupStats = null; + DataDownloadFileGroupStats fileGroupStats = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(groupKey.getGroupName()) + .setFileGroupVersionNumber(fileGroupVersionNumber) + .setBuildId(buildId) + .setVariantId(variantId) + .setOwnerPackage(groupKey.getOwnerPackage()) + .build(); eventLogger.logMddNetworkSavings( fileGroupStats, 0, fullFileSize, downloadedFileSize, dataFile.getFileId(), - /* deltaIndex = */ 0); + /* deltaIndex= */ 0); } } fileStorage.deleteFile(source); @@ -303,7 +312,14 @@ public class DownloaderCallbackImpl implements DownloaderCallback { .build(); } try { - Void fileGroupStats = null; + DataDownloadFileGroupStats fileGroupStats = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(groupKey.getGroupName()) + .setFileGroupVersionNumber(fileGroupVersionNumber) + .setBuildId(buildId) + .setVariantId(variantId) + .setOwnerPackage(groupKey.getOwnerPackage()) + .build(); eventLogger.logMddNetworkSavings( fileGroupStats, 0, @@ -387,7 +403,7 @@ public class DownloaderCallbackImpl implements DownloaderCallback { "%s: Checksum mismatch detected but the has already reached retry limit!" + " Skipping removal for file %s", TAG, checksum); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); } else { LogUtil.d( "%s: Removing file and marking as corrupted due to checksum mismatch", TAG); diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/downloader/MddFileDownloader.java b/java/com/google/android/libraries/mobiledatadownload/internal/downloader/MddFileDownloader.java index b1de88b..1c23a97 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/downloader/MddFileDownloader.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/downloader/MddFileDownloader.java @@ -15,6 +15,9 @@ */ package com.google.android.libraries.mobiledatadownload.internal.downloader; +import static com.google.common.util.concurrent.Futures.immediateFailedFuture; +import static com.google.common.util.concurrent.Futures.immediateFuture; +import static com.google.common.util.concurrent.Futures.immediateVoidFuture; import static java.lang.Math.min; import android.content.Context; @@ -35,14 +38,18 @@ import com.google.android.libraries.mobiledatadownload.internal.ApplicationConte import com.google.android.libraries.mobiledatadownload.internal.annotations.SequentialControlExecutor; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.android.libraries.mobiledatadownload.internal.logging.LoggingStateStore; +import com.google.android.libraries.mobiledatadownload.internal.util.DownloadFutureMap; +import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupUtil; import com.google.android.libraries.mobiledatadownload.monitor.DownloadProgressMonitor; import com.google.android.libraries.mobiledatadownload.monitor.NetworkUsageMonitor; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentFuture; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Optional; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.FluentFuture; -import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.AsyncFunction; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListenableFutureTask; import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions; import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions.DeviceNetworkPolicy; import com.google.mobiledatadownload.internal.MetadataProto.ExtraHttpHeader; @@ -80,8 +87,18 @@ public class MddFileDownloader { private final Flags flags; // Cache for all on-going downloads. This will be used to de-dup download requests. + // NOTE: all operations are internally sequenced through an ExecutionSequencer. + // NOTE: this map and fileUriToDownloadFutureMap are mutually exclusive and the use of + // one or the other is based on an MDD feature flag (enableFileDownloadDedupByFileKey). Once the + // flag is fully rolled out, this map will be used exclusively. + private final DownloadFutureMap<Void> downloadOrCopyFutureMap; + + // Cache for all on-going downloads. This will be used to de-dup download requests. // NOTE: currently we assume that this map will only be accessed through the // SequentialControlExecutor, so we don't need synchronization here. + // NOTE: this map and downloadOrCopyFutureMap are mutually exclusive and the use of + // one or the other is based on an MDD feature flag (enableFileDownloadDedupByFileKey). Once the + // flag is fully rolled out, this map will not be used. @VisibleForTesting final HashMap<Uri, ListenableFuture<Void>> fileUriToDownloadFutureMap = new HashMap<>(); @@ -103,14 +120,17 @@ public class MddFileDownloader { this.loggingStateStore = loggingStateStore; this.sequentialControlExecutor = sequentialControlExecutor; this.flags = flags; + this.downloadOrCopyFutureMap = DownloadFutureMap.create(sequentialControlExecutor); } /** * Start downloading the file. * + * @param fileKey key that identifies the shared file to download. * @param groupKey GroupKey that contains the file to download. * @param fileGroupVersionNumber version number of the group that contains the file to download. * @param buildId build id of the group that contains the file to download. + * @param variantId variant id of the group that contains the file to download. * @param fileUri - the File Uri to download the file at. * @param urlToDownload - The url of the file to download. * @param fileSize - the expected size of the file to download. @@ -121,9 +141,11 @@ public class MddFileDownloader { * @return - ListenableFuture representing the download result of a file. */ public ListenableFuture<Void> startDownloading( + String fileKey, GroupKey groupKey, int fileGroupVersionNumber, long buildId, + String variantId, Uri fileUri, String urlToDownload, int fileSize, @@ -131,77 +153,132 @@ public class MddFileDownloader { DownloaderCallback callback, int trafficTag, List<ExtraHttpHeader> extraHttpHeaders) { - if (fileUriToDownloadFutureMap.containsKey(fileUri)) { - return fileUriToDownloadFutureMap.get(fileUri); - } - return addCallbackAndRegister( - fileUri, - callback, - startDownloadingInternal( - groupKey, - fileGroupVersionNumber, - buildId, - fileUri, - urlToDownload, - fileSize, - downloadConditions, - trafficTag, - extraHttpHeaders)); + return PropagatedFutures.transformAsync( + getInProgressFuture(fileKey, fileUri), + inProgressFuture -> { + if (inProgressFuture.isPresent()) { + return inProgressFuture.get(); + } + return addCallbackAndRegister( + fileKey, + fileUri, + callback, + unused -> + startDownloadingInternal( + groupKey, + fileGroupVersionNumber, + buildId, + variantId, + fileUri, + urlToDownload, + fileSize, + downloadConditions, + trafficTag, + extraHttpHeaders)); + }, + sequentialControlExecutor); } /** * Adds Callback to given Future and Registers future in in-progress cache. * - * <p>Contains shared logic of connecting {@code callback} to {@code downloadOrCopyFuture} and + * <p>Contains shared logic of connecting {@code callback} to {@code downloadOrCopyFunction} and * registers future in the internal in-progress cache. This cache allows similar download/copy * requests to be deduped instead of being performed twice. * * <p>NOTE: this method assumes the cache has already been checked for an in-progress operation * and no in-progress operation exists for {@code fileUri}. * + * @param fileKey key that identifies the shared file. * @param fileUri the destination of the download/copy (used as Key in in-progress cache) * @param callback the callback that should be run after the given download/copy future - * @param downloadOrCopyFuture a ListenableFuture that will perform the download/copy + * @param downloadOrCopyFunction an AsyncFunction that will perform the download/copy * @return a ListenableFuture that calls the correct callback after {@code downloadOrCopyFuture * completes} */ private ListenableFuture<Void> addCallbackAndRegister( - Uri fileUri, DownloaderCallback callback, ListenableFuture<Void> downloadOrCopyFuture) { + String fileKey, + Uri fileUri, + DownloaderCallback callback, + AsyncFunction<Void, Void> downloadOrCopyFunction) { + // Use ListenableFutureTask to create a future without starting it. This ensures we can + // successfully add our future to download/copy before the operation starts. + ListenableFutureTask<Void> startTask = ListenableFutureTask.create(() -> null); + // Use transform & catching to ensure that we correctly chain everything. - FluentFuture<Void> transformedFuture = - FluentFuture.from(downloadOrCopyFuture) + PropagatedFluentFuture<Void> downloadOrCopyFuture = + PropagatedFluentFuture.from(startTask) + .transformAsync(downloadOrCopyFunction, sequentialControlExecutor) .transformAsync( voidArg -> callback.onDownloadComplete(fileUri), sequentialControlExecutor /*Run callbacks on @SequentialControlExecutor*/) .catchingAsync( - DownloadException.class, + Exception.class, e -> - Futures.transformAsync( - callback.onDownloadFailed(e), + // Rethrow exception so the failure is passed back up the future chain. + PropagatedFutures.transformAsync( + callback.onDownloadFailed(asDownloadException(e)), voidArg -> { throw e; }, sequentialControlExecutor), sequentialControlExecutor /*Run callbacks on @SequentialControlExecutor*/); - fileUriToDownloadFutureMap.put(fileUri, transformedFuture); + // Add this future to the future map, then start startTask to unblock download/copy. The order + // ensures that the download/copy happens only if we were able to add the future to the map. + PropagatedFluentFuture<Void> transformedFuture = + PropagatedFluentFuture.from(addFutureToMap(downloadOrCopyFuture, fileKey, fileUri)) + .transformAsync( + unused -> { + startTask.run(); + return downloadOrCopyFuture; + }, + sequentialControlExecutor); - // We want to remove the transformedFuture from the cache when the transformedFuture finishes. + // We want to remove the future from the cache when the transformedFuture finishes. // However there may be a race condition and transformedFuture may finish before we put it into // the cache. // To prevent this race condition, we add a callback to transformedFuture to make sure the // removal happens after the putting it in the map. // A transform would not work since we want to run the removal even when the transform failed. transformedFuture.addListener( - () -> fileUriToDownloadFutureMap.remove(fileUri), sequentialControlExecutor); + () -> { + ListenableFuture<Void> unused = removeFutureFromMap(fileKey, fileUri); + }, + sequentialControlExecutor); return transformedFuture; } + private ListenableFuture<Void> addFutureToMap( + ListenableFuture<Void> downloadOrCopyFuture, String fileKey, Uri fileUri) { + if (!flags.enableFileDownloadDedupByFileKey()) { + fileUriToDownloadFutureMap.put(fileUri, downloadOrCopyFuture); + return immediateVoidFuture(); + } else { + return downloadOrCopyFutureMap.add(fileKey, downloadOrCopyFuture); + } + } + + private ListenableFuture<Void> removeFutureFromMap(String fileKey, Uri fileUri) { + if (!flags.enableFileDownloadDedupByFileKey()) { + // Return the removed future if it exists, otherwise return immediately (Extra check added to + // satisfy nullness checker). + ListenableFuture<Void> removedFuture = fileUriToDownloadFutureMap.remove(fileUri); + if (removedFuture != null) { + return removedFuture; + } + return immediateVoidFuture(); + } else { + return downloadOrCopyFutureMap.remove(fileKey); + } + } + private ListenableFuture<Void> startDownloadingInternal( GroupKey groupKey, int fileGroupVersionNumber, long buildId, + String variantId, Uri fileUri, String urlToDownload, int fileSize, @@ -212,7 +289,7 @@ public class MddFileDownloader { && flags.downloaderEnforceHttps() && !urlToDownload.startsWith("https")) { LogUtil.e("%s: File url = %s is not secure", TAG, urlToDownload); - return Futures.immediateFailedFuture( + return immediateFailedFuture( DownloadException.builder() .setDownloadResultCode(DownloadResultCode.INSECURE_URL_ERROR) .build()); @@ -227,16 +304,17 @@ public class MddFileDownloader { } try { - checkStorageConstraints(context, fileSize - currentFileSize, downloadConditions, flags); + checkStorageConstraints( + context, urlToDownload, fileSize - currentFileSize, downloadConditions, flags); } catch (DownloadException e) { // Wrap exception in future to break future chain. LogUtil.e("%s: Not enough space to download file %s", TAG, urlToDownload); - return Futures.immediateFailedFuture(e); + return immediateFailedFuture(e); } if (flags.logNetworkStats()) { networkUsageMonitor.monitorUri( - fileUri, groupKey, buildId, fileGroupVersionNumber, loggingStateStore); + fileUri, groupKey, buildId, variantId, fileGroupVersionNumber, loggingStateStore); } else { LogUtil.w("%s: NetworkUsageMonitor is disabled", TAG); } @@ -273,8 +351,29 @@ public class MddFileDownloader { } /** + * Gets an in-progress future (if it exists), otherwise returns absent. + * + * <p>This method allows easier deduplication of file downloads/copies, by allowing callers to + * query against the internal download future map. This method is assumed to be called when a + * SharedFile state is DOWNLOAD_IN_PROGRESS. + * + * @param fileKey key that identifies the shared file. + * @param fileUri - the File Uri to download the file at. + * @return - ListenableFuture representing an in-progress download/copy for the given file. + */ + public ListenableFuture<Optional<ListenableFuture<Void>>> getInProgressFuture( + String fileKey, Uri fileUri) { + if (!flags.enableFileDownloadDedupByFileKey()) { + return immediateFuture(Optional.fromNullable(fileUriToDownloadFutureMap.get(fileUri))); + } else { + return downloadOrCopyFutureMap.get(fileKey); + } + } + + /** * Start Copying a file to internal storage * + * @param fileKey key that identifies the shared file to copy. * @param fileUri the File Uri where content should be copied. * @param urlToDownload the url to copy, should be inlinefile: scheme. * @param fileSize the size of the file to copy. @@ -284,20 +383,28 @@ public class MddFileDownloader { * @return ListenableFuture representing the result of a file copy. */ public ListenableFuture<Void> startCopying( + String fileKey, Uri fileUri, String urlToDownload, int fileSize, @Nullable DownloadConditions downloadConditions, DownloaderCallback downloaderCallback, FileSource inlineFileSource) { - if (fileUriToDownloadFutureMap.containsKey(fileUri)) { - return fileUriToDownloadFutureMap.get(fileUri); - } - return addCallbackAndRegister( - fileUri, - downloaderCallback, - startCopyingInternal( - fileUri, urlToDownload, fileSize, downloadConditions, inlineFileSource)); + return PropagatedFutures.transformAsync( + getInProgressFuture(fileKey, fileUri), + inProgressFuture -> { + if (inProgressFuture.isPresent()) { + return inProgressFuture.get(); + } + return addCallbackAndRegister( + fileKey, + fileUri, + downloaderCallback, + unused -> + startCopyingInternal( + fileUri, urlToDownload, fileSize, downloadConditions, inlineFileSource)); + }, + sequentialControlExecutor); } private ListenableFuture<Void> startCopyingInternal( @@ -307,12 +414,24 @@ public class MddFileDownloader { @Nullable DownloadConditions downloadConditions, FileSource inlineFileSource) { + int finalFileSize = fileSize; + if (inlineFileSource.getKind().equals(FileSource.Kind.BYTESTRING)) { + int sourceFileSize = inlineFileSource.byteString().size(); + if (sourceFileSize != fileSize) { + LogUtil.w( + "%s: expected file size (%d) does not match source file size (%d) -- using source file" + + " size for storage check; file: %s", + TAG, fileSize, sourceFileSize, urlToCopy); + finalFileSize = sourceFileSize; + } + } + try { - checkStorageConstraints(context, fileSize, downloadConditions, flags); + checkStorageConstraints(context, urlToCopy, finalFileSize, downloadConditions, flags); } catch (DownloadException e) { // Wrap exception in future to break future chain. LogUtil.e("%s: Not enough space to download file %s", TAG, urlToCopy); - return Futures.immediateFailedFuture(e); + return immediateFailedFuture(e); } // TODO(b/177361344): Only monitor file if download listener is supported @@ -332,17 +451,24 @@ public class MddFileDownloader { /** * Stop downloading the file. * + * @param fileKey - key that identifies the file to stop downloading. * @param fileUri - the File Uri of the file to stop downloading. */ - public void stopDownloading(Uri fileUri) { - ListenableFuture<Void> pendingDownloadFuture = fileUriToDownloadFutureMap.get(fileUri); - if (pendingDownloadFuture != null) { - LogUtil.d("%s: Cancel download file %s", TAG, fileUri); - fileUriToDownloadFutureMap.remove(fileUri); - pendingDownloadFuture.cancel(true); - } else { - LogUtil.w("%s: stopDownloading on non-existent download", TAG); - } + public void stopDownloading(String fileKey, Uri fileUri) { + ListenableFuture<Void> unused = + PropagatedFutures.transformAsync( + getInProgressFuture(fileKey, fileUri), + inProgressFuture -> { + if (inProgressFuture.isPresent()) { + LogUtil.d("%s: Cancel download file %s", TAG, fileUri); + inProgressFuture.get().cancel(/* mayInterruptIfRunning= */ true); + return removeFutureFromMap(fileKey, fileUri); + } else { + LogUtil.w("%s: stopDownloading on non-existent download", TAG); + return immediateVoidFuture(); + } + }, + sequentialControlExecutor); } /** @@ -363,14 +489,15 @@ public class MddFileDownloader { * @throws DownloadException when storing a file with the given size would hit the given storage * thresholds */ - public static void checkStorageConstraints( + private static void checkStorageConstraints( Context context, + String url, long bytesNeeded, @Nullable DownloadConditions downloadConditions, Flags flags) throws DownloadException { if (flags.enforceLowStorageBehavior() - && !shouldDownload(context, bytesNeeded, downloadConditions, flags)) { + && !shouldDownload(context, url, bytesNeeded, downloadConditions, flags)) { throw DownloadException.builder() .setDownloadResultCode(DownloadResultCode.LOW_DISK_ERROR) .build(); @@ -385,9 +512,15 @@ public class MddFileDownloader { */ private static boolean shouldDownload( Context context, + String url, long bytesNeeded, @Nullable DownloadConditions downloadConditions, Flags flags) { + // If we are using a placeholder (inline file + 0 byte size), bypass storage checks. + if (FileGroupUtil.isInlineFile(url) && bytesNeeded == 0L) { + return true; + } + StatFs stats = new StatFs(context.getFilesDir().getAbsolutePath()); long totalBytes = (long) stats.getBlockCount() * stats.getBlockSize(); @@ -421,6 +554,23 @@ public class MddFileDownloader { return remainingBytesAfterDownload > minBytes; } + /** + * Wraps throwable as DownloadException if it isn't one already. + * + * <p>This method doesn't check the incoming throwable besides the type and defaults the download + * result code to UNKNOWN_ERROR. + */ + private static DownloadException asDownloadException(Throwable t) { + if (t instanceof DownloadException) { + return (DownloadException) t; + } + + return DownloadException.builder() + .setCause(t) + .setDownloadResultCode(DownloadResultCode.UNKNOWN_ERROR) + .build(); + } + /** Interface called by the downloader when download either completes or fails. */ public static interface DownloaderCallback { /** Called on download complete. */ diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/experimentation/BUILD b/java/com/google/android/libraries/mobiledatadownload/internal/experimentation/BUILD index ffa6fc9..e6857e1 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/experimentation/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/internal/experimentation/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -25,6 +26,7 @@ android_library( srcs = ["DownloadStageManager.java"], deps = [ "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "@androidx_annotation_annotation", "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], @@ -36,6 +38,7 @@ android_library( deps = [ ":DownloadStageManager", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "@androidx_annotation_annotation", "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/BUILD b/java/com/google/android/libraries/mobiledatadownload/internal/logging/BUILD index 8ea9550..6ce86c3 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -31,6 +32,8 @@ android_library( name = "EventLogger", srcs = ["EventLogger.java"], deps = [ + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "@com_google_auto_value", "@com_google_guava_guava", ], @@ -41,6 +44,8 @@ android_library( srcs = ["NoOpEventLogger.java"], deps = [ ":EventLogger", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "@com_google_guava_guava", ], ) @@ -53,8 +58,12 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/internal:FileGroupManager", "//java/com/google/android/libraries/mobiledatadownload/internal:FileGroupsMetadata", "//java/com/google/android/libraries/mobiledatadownload/internal/annotations:SequentialControlExecutor", + "//java/com/google/android/libraries/mobiledatadownload/internal/collect", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "//java/com/google/android/libraries/mobiledatadownload/internal/util:FileGroupUtil", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "@com_google_guava_guava", "@javax_inject", ], @@ -67,7 +76,10 @@ android_library( ], deps = [ ":EventLogger", + ":LogUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "@androidx_annotation_annotation", "@com_google_errorprone_error_prone_annotations", ], @@ -79,13 +91,15 @@ android_library( "MddEventLogger.java", ], deps = [ - ":EventLogger", - ":LogSampler", - ":LogUtil", - ":LoggingStateStore", "//java/com/google/android/libraries/mobiledatadownload:Flags", "//java/com/google/android/libraries/mobiledatadownload:Logger", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogSampler", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LoggingStateStore", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "@com_google_guava_guava", ], ) @@ -99,6 +113,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload:SilentFeedback", "//java/com/google/android/libraries/mobiledatadownload/annotations", "//java/com/google/android/libraries/mobiledatadownload/file", + "//java/com/google/android/libraries/mobiledatadownload/file/openers:recursive_size", "//java/com/google/android/libraries/mobiledatadownload/internal:ApplicationContext", "//java/com/google/android/libraries/mobiledatadownload/internal:FileGroupsMetadata", "//java/com/google/android/libraries/mobiledatadownload/internal:MddConstants", @@ -106,10 +121,12 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/internal:SharedFileManager", "//java/com/google/android/libraries/mobiledatadownload/internal:SharedFilesMetadata", "//java/com/google/android/libraries/mobiledatadownload/internal/annotations:SequentialControlExecutor", + "//java/com/google/android/libraries/mobiledatadownload/internal/collect", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "//java/com/google/android/libraries/mobiledatadownload/internal/util:DirectoryUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/util:FileGroupUtil", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", - "@com_google_auto_value", + "//proto:logs_java_proto_lite", "@com_google_guava_guava", "@javax_inject", ], @@ -120,12 +137,13 @@ android_library( srcs = ["NetworkLogger.java"], deps = [ ":EventLogger", - ":LoggingStateStore", "//java/com/google/android/libraries/mobiledatadownload:Flags", "//java/com/google/android/libraries/mobiledatadownload/annotations", "//java/com/google/android/libraries/mobiledatadownload/internal:ApplicationContext", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LoggingStateStore", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "//proto:logs_java_proto_lite", "@com_google_guava_guava", "@javax_inject", ], @@ -149,7 +167,10 @@ android_library( ":LogUtil", ":LoggingStateStore", "//java/com/google/android/libraries/mobiledatadownload:Flags", + "//java/com/google/android/libraries/mobiledatadownload/tracing", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "//java/com/google/protobuf/util:time_lite", + "//proto:logs_java_proto_lite", "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], @@ -166,3 +187,23 @@ android_library( "@com_google_guava_guava", ], ) + +android_library( + name = "SharedPreferencesLoggingState", + srcs = [ + "SharedPreferencesLoggingState.java", + ], + deps = [ + ":LoggingStateStore", + "//google/protobuf:timestamp_java_proto_lite", + "//java/com/google/android/libraries/mobiledatadownload:TimeSource", + "//java/com/google/android/libraries/mobiledatadownload/internal:MddConstants", + "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "//java/com/google/android/libraries/mobiledatadownload/internal/util:FileGroupsMetadataUtil", + "//java/com/google/android/libraries/mobiledatadownload/internal/util:SharedPreferencesUtil", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "//java/com/google/protobuf/util:time_lite", + "@androidx_annotation_annotation", + "@com_google_guava_guava", + ], +) diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/DownloadStateLogger.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/DownloadStateLogger.java index a8f388f..85b13a9 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/DownloadStateLogger.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/DownloadStateLogger.java @@ -15,8 +15,9 @@ */ package com.google.android.libraries.mobiledatadownload.internal.logging; -import androidx.annotation.VisibleForTesting; import com.google.errorprone.annotations.CheckReturnValue; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupBookkeeping; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; @@ -25,8 +26,8 @@ import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInterna public final class DownloadStateLogger { private static final String TAG = "FileGroupStatusLogger"; - @VisibleForTesting - enum Operation { + /** The type of operation for which the logger will log events. */ + public enum Operation { DOWNLOAD, IMPORT, }; @@ -47,13 +48,18 @@ public final class DownloadStateLogger { return new DownloadStateLogger(eventLogger, Operation.IMPORT); } + /** Gets the operation associated with this logger. */ + public Operation getOperation() { + return operation; + } + public void logStarted(DataFileGroupInternal fileGroup) { switch (operation) { case DOWNLOAD: - logEventWithDataFileGroup(0, fileGroup); + logEventWithDataFileGroup(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, fileGroup); break; case IMPORT: - logEventWithDataFileGroup(0, fileGroup); + logEventWithDataFileGroup(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, fileGroup); break; } } @@ -61,10 +67,10 @@ public final class DownloadStateLogger { public void logPending(DataFileGroupInternal fileGroup) { switch (operation) { case DOWNLOAD: - logEventWithDataFileGroup(0, fileGroup); + logEventWithDataFileGroup(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, fileGroup); break; case IMPORT: - logEventWithDataFileGroup(0, fileGroup); + logEventWithDataFileGroup(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, fileGroup); break; } } @@ -72,10 +78,10 @@ public final class DownloadStateLogger { public void logFailed(DataFileGroupInternal fileGroup) { switch (operation) { case DOWNLOAD: - logEventWithDataFileGroup(0, fileGroup); + logEventWithDataFileGroup(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, fileGroup); break; case IMPORT: - logEventWithDataFileGroup(0, fileGroup); + logEventWithDataFileGroup(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, fileGroup); break; } } @@ -83,11 +89,11 @@ public final class DownloadStateLogger { public void logComplete(DataFileGroupInternal fileGroup) { switch (operation) { case DOWNLOAD: - logEventWithDataFileGroup(0, fileGroup); + logEventWithDataFileGroup(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, fileGroup); logDownloadLatency(fileGroup); break; case IMPORT: - logEventWithDataFileGroup(0, fileGroup); + logEventWithDataFileGroup(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, fileGroup); break; } } @@ -99,7 +105,15 @@ public final class DownloadStateLogger { return; } - Void fileGroupDetails = null; + DataDownloadFileGroupStats fileGroupDetails = + DataDownloadFileGroupStats.newBuilder() + .setOwnerPackage(fileGroup.getOwnerPackage()) + .setFileGroupName(fileGroup.getGroupName()) + .setFileGroupVersionNumber(fileGroup.getFileGroupVersionNumber()) + .setFileCount(fileGroup.getFileCount()) + .setBuildId(fileGroup.getBuildId()) + .setVariantId(fileGroup.getVariantId()) + .build(); DataFileGroupBookkeeping bookkeeping = fileGroup.getBookkeeping(); long newFilesReceivedTimestamp = bookkeeping.getGroupNewFilesReceivedTimestamp(); @@ -111,7 +125,8 @@ public final class DownloadStateLogger { eventLogger.logMddDownloadLatency(fileGroupDetails, downloadLatency); } - private void logEventWithDataFileGroup(int code, DataFileGroupInternal fileGroup) { + private void logEventWithDataFileGroup( + MddClientEvent.Code code, DataFileGroupInternal fileGroup) { eventLogger.logEventSampled( code, fileGroup.getGroupName(), diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/EventLogger.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/EventLogger.java index e1ed276..83e8311 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/EventLogger.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/EventLogger.java @@ -18,17 +18,22 @@ package com.google.android.libraries.mobiledatadownload.internal.logging; import com.google.auto.value.AutoValue; import com.google.common.util.concurrent.AsyncCallable; import com.google.common.util.concurrent.ListenableFuture; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; +import com.google.mobiledatadownload.LogEnumsProto.MddDownloadResult; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; +import com.google.mobiledatadownload.LogProto.MddFileGroupStatus; +import com.google.mobiledatadownload.LogProto.MddStorageStats; import java.util.List; /** Interface for remote logging. */ public interface EventLogger { /** Log an mdd event */ - void logEventSampled(int eventCode); + void logEventSampled(MddClientEvent.Code eventCode); /** Log an mdd event with an associated file group. */ void logEventSampled( - int eventCode, + MddClientEvent.Code eventCode, String fileGroupName, int fileGroupVersionNumber, long buildId, @@ -38,7 +43,7 @@ public interface EventLogger { * Log an mdd event. This not sampled. Caller should make sure this method is called after * sampling at the passed in value of sample interval. */ - void logEventAfterSample(int eventCode, int sampleInterval); + void logEventAfterSample(MddClientEvent.Code eventCode, int sampleInterval); /** * Log mdd file group stats. The buildFileGroupStats callable is only called if the event is going @@ -55,28 +60,31 @@ public interface EventLogger { /** Simple wrapper class for MDD file group stats and details. */ @AutoValue abstract class FileGroupStatusWithDetails { - abstract Void fileGroupStatus(); + abstract MddFileGroupStatus fileGroupStatus(); - abstract Void fileGroupDetails(); + abstract DataDownloadFileGroupStats fileGroupDetails(); - static FileGroupStatusWithDetails create(Void fileGroupStatus, Void fileGroupDetails) { + static FileGroupStatusWithDetails create( + MddFileGroupStatus fileGroupStatus, DataDownloadFileGroupStats fileGroupDetails) { return new AutoValue_EventLogger_FileGroupStatusWithDetails( fileGroupStatus, fileGroupDetails); } } /** Log mdd api call stats. */ - void logMddApiCallStats(Void fileGroupDetails, Void apiCallStats); + void logMddApiCallStats(DataDownloadFileGroupStats fileGroupDetails, Void apiCallStats); + + void logMddLibApiResultLog(Void mddLibApiResultLog); /** * Log mdd storage stats. The buildMddStorageStats callable is only called if the event is going * to be logged. * - * @param buildMddStorageStats callable which builds the Void to log. + * @param buildMddStorageStats callable which builds the MddStorageStats to log. * @return a future that completes when the logging work is done. The future will complete with a * failure if the callable fails or if there is an error when logging. */ - ListenableFuture<Void> logMddStorageStats(AsyncCallable<Void> buildMddStorageStats); + ListenableFuture<Void> logMddStorageStats(AsyncCallable<MddStorageStats> buildMddStorageStats); /** * Log mdd network stats. The buildMddNetworkStats callable is only called if the event is going @@ -93,7 +101,7 @@ public interface EventLogger { /** Log the network savings of MDD download features */ void logMddNetworkSavings( - Void fileGroupDetails, + DataDownloadFileGroupStats fileGroupDetails, int code, long fullFileSize, long downloadedFileSize, @@ -101,17 +109,22 @@ public interface EventLogger { int deltaIndex); /** Log mdd download result events. */ - void logMddDownloadResult(int code, Void fileGroupDetails); + void logMddDownloadResult( + MddDownloadResult.Code code, DataDownloadFileGroupStats fileGroupDetails); /** Log stats of mdd {@code getFileGroup} and {@code getFileGroupByFilter} calls. */ - void logMddQueryStats(Void fileGroupDetails); + void logMddQueryStats(DataDownloadFileGroupStats fileGroupDetails); /** Log mdd stats on android sharing events. */ void logMddAndroidSharingLog(Void event); /** Log mdd download latency. */ - void logMddDownloadLatency(Void fileGroupStats, Void downloadLatency); + void logMddDownloadLatency(DataDownloadFileGroupStats fileGroupStats, Void downloadLatency); /** Log mdd usage event. */ - void logMddUsageEvent(Void fileGroupDetails, Void usageEventLog); + void logMddUsageEvent(DataDownloadFileGroupStats fileGroupDetails, Void usageEventLog); + + /** Log new config received event. */ + void logNewConfigReceived( + DataDownloadFileGroupStats fileGroupDetails, Void newConfigReceivedInfo); } diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/FileGroupStatsLogger.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/FileGroupStatsLogger.java index 3803c33..3e10eaa 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/FileGroupStatsLogger.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/FileGroupStatsLogger.java @@ -15,13 +15,20 @@ */ package com.google.android.libraries.mobiledatadownload.internal.logging; -import android.util.Pair; +import static com.google.common.util.concurrent.Futures.immediateFuture; + import com.google.android.libraries.mobiledatadownload.internal.FileGroupManager; +import com.google.android.libraries.mobiledatadownload.internal.FileGroupManager.GroupDownloadStatus; import com.google.android.libraries.mobiledatadownload.internal.FileGroupsMetadata; import com.google.android.libraries.mobiledatadownload.internal.annotations.SequentialControlExecutor; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup; +import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupUtil; import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.mobiledatadownload.LogEnumsProto.MddFileGroupDownloadStatus; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; +import com.google.mobiledatadownload.LogProto.MddFileGroupStatus; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; import java.util.ArrayList; @@ -65,14 +72,24 @@ public class FileGroupStatsLogger { downloadedAndPendingGroups -> { List<ListenableFuture<EventLogger.FileGroupStatusWithDetails>> futures = new ArrayList<>(); - for (Pair<GroupKey, DataFileGroupInternal> pair : downloadedAndPendingGroups) { - GroupKey groupKey = pair.first; - DataFileGroupInternal dataFileGroup = pair.second; + for (GroupKeyAndGroup pair : downloadedAndPendingGroups) { + GroupKey groupKey = pair.groupKey(); + DataFileGroupInternal dataFileGroup = pair.dataFileGroup(); if (dataFileGroup == null) { continue; } - Void fileGroupDetails = null; + DataDownloadFileGroupStats fileGroupDetails = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(groupKey.getGroupName()) + .setOwnerPackage(groupKey.getOwnerPackage()) + .setFileGroupVersionNumber(dataFileGroup.getFileGroupVersionNumber()) + .setFileCount(dataFileGroup.getFileCount()) + .setInlineFileCount(FileGroupUtil.getInlineFileCount(dataFileGroup)) + .setHasAccount(!groupKey.getAccount().isEmpty()) + .setBuildId(dataFileGroup.getBuildId()) + .setVariantId(dataFileGroup.getVariantId()) + .build(); futures.add( PropagatedFutures.transform( @@ -87,8 +104,42 @@ public class FileGroupStatsLogger { sequentialControlExecutor); } - private ListenableFuture<Void> buildFileGroupStatus( + private ListenableFuture<MddFileGroupStatus> buildFileGroupStatus( DataFileGroupInternal dataFileGroup, GroupKey groupKey, int daysSinceLastLog) { - return Futures.immediateVoidFuture(); + MddFileGroupStatus.Builder fileGroupStatus = + MddFileGroupStatus.newBuilder().setDaysSinceLastLog(daysSinceLastLog); + if (dataFileGroup.getBookkeeping().hasGroupNewFilesReceivedTimestamp()) { + fileGroupStatus.setGroupAddedTimestampInSeconds( + dataFileGroup.getBookkeeping().getGroupNewFilesReceivedTimestamp() / 1000); + } else { + fileGroupStatus.setGroupAddedTimestampInSeconds(-1); + } + + if (groupKey.getDownloaded()) { + fileGroupStatus.setFileGroupDownloadStatus(MddFileGroupDownloadStatus.Code.COMPLETE); + if (dataFileGroup.getBookkeeping().hasGroupDownloadedTimestampInMillis()) { + fileGroupStatus.setGroupDownloadedTimestampInSeconds( + dataFileGroup.getBookkeeping().getGroupDownloadedTimestampInMillis() / 1000); + } else { + fileGroupStatus.setGroupDownloadedTimestampInSeconds(-1); + } + return immediateFuture(fileGroupStatus.build()); + } else { + fileGroupStatus.setGroupDownloadedTimestampInSeconds(-1); + return PropagatedFutures.transform( + fileGroupManager.getFileGroupDownloadStatus(dataFileGroup), + status -> { + if (status == GroupDownloadStatus.DOWNLOADED || status == GroupDownloadStatus.PENDING) { + // Log pending even if verify returns downloaded, as it will be marked as + // completed in the next periodic task. + fileGroupStatus.setFileGroupDownloadStatus(MddFileGroupDownloadStatus.Code.PENDING); + } else { + // TODO(b/73490689): Log the reason for failure along with this. + fileGroupStatus.setFileGroupDownloadStatus(MddFileGroupDownloadStatus.Code.FAILED); + } + return fileGroupStatus.build(); + }, + sequentialControlExecutor); + } } } diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/LogSampler.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/LogSampler.java index 212dec5..b25028c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/LogSampler.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/LogSampler.java @@ -23,111 +23,131 @@ import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentF import com.google.common.base.Optional; import com.google.common.util.concurrent.ListenableFuture; import com.google.errorprone.annotations.CheckReturnValue; +import com.google.mobiledatadownload.LogProto.StableSamplingInfo; + import java.util.Random; /** Class responsible for sampling events. */ @CheckReturnValue public final class LogSampler { - private final Flags flags; - private final Random random; + private final Flags flags; + private final Random random; - /** - * Construct the log sampler. - * - * @param flags used to check whether stable sampling is enabled. - * @param random used to generate random numbers for event based sampling only. - */ - public LogSampler(Flags flags, Random random) { - this.flags = flags; - this.random = random; - } + /** + * Construct the log sampler. + * + * @param flags used to check whether stable sampling is enabled. + * @param random used to generate random numbers for event based sampling only. + */ + public LogSampler(Flags flags, Random random) { + this.flags = flags; + this.random = random; + } - /** - * Determines whether the event should be logged. If the event should be logged it returns an - * instance of Void that should be attached to the log events. - * - * <p>If stable sampling is enabled, this is deterministic. If stable sampling is disabled, the - * result can change on each call based on the provided Random instance. - * - * @param sampleInterval the inverse sampling rate to use. This is controlled by flags per - * event-type. For stable sampling it's expected that 100 % sampleInterval == 0. - * @param loggingStateStore used to read persisted random number when stable sampling is enabled. - * If it is absent, stable sampling will not be used. - * @return a future of an optional of StableSamplingInfo. The future will resolve to an absent - * Optional if the event should not be logged. If the event should be logged, the returned - * Void should be attached to the log event. - */ - public ListenableFuture<Optional<Void>> shouldLog( - long sampleInterval, Optional<LoggingStateStore> loggingStateStore) { - if (sampleInterval == 0L) { - return immediateFuture(Optional.absent()); - } else if (sampleInterval < 0L) { - LogUtil.e("Bad sample interval (negative number): %d", sampleInterval); - return immediateFuture(Optional.absent()); - } else if (flags.enableRngBasedDeviceStableSampling() && loggingStateStore.isPresent()) { - return shouldLogDeviceStable(sampleInterval, loggingStateStore.get()); - } else { - return shouldLogPerEvent(sampleInterval); + /** + * Determines whether the event should be logged. If the event should be logged it returns an + * instance of StableSamplingInfo that should be attached to the log events. + * + * <p>If stable sampling is enabled, this is deterministic. If stable sampling is disabled, the + * result can change on each call based on the provided Random instance. + * + * @param sampleInterval the inverse sampling rate to use. This is controlled by flags per + * event-type. For stable sampling it's expected that 100 % + * sampleInterval == 0. + * @param loggingStateStore used to read persisted random number when stable sampling is + * enabled. + * If it is absent, stable sampling will not be used. + * @return a future of an optional of StableSamplingInfo. The future will resolve to an absent + * Optional if the event should not be logged. If the event should be logged, the returned + * StableSamplingInfo should be attached to the log event. + */ + public ListenableFuture<Optional<StableSamplingInfo>> shouldLog( + long sampleInterval, Optional<LoggingStateStore> loggingStateStore) { + if (sampleInterval == 0L) { + return immediateFuture(Optional.absent()); + } else if (sampleInterval < 0L) { + LogUtil.e("Bad sample interval (negative number): %d", sampleInterval); + return immediateFuture(Optional.absent()); + } else if (flags.enableRngBasedDeviceStableSampling() && loggingStateStore.isPresent()) { + return shouldLogDeviceStable(sampleInterval, loggingStateStore.get()); + } else { + return shouldLogPerEvent(sampleInterval); + } } - } - /** - * Returns standard random event based sampling. - * - * @return if the event should be sampled, returns the Void with stable_sampling_used = false. - * Otherwise, returns an empty Optional. - */ - private ListenableFuture<Optional<Void>> shouldLogPerEvent(long sampleInterval) { - if (shouldSamplePerEvent(sampleInterval)) { - return immediateFuture(Optional.absent()); - } else { - return immediateFuture(Optional.absent()); + /** + * Returns standard random event based sampling. + * + * @return if the event should be sampled, returns the StableSamplingInfo with + * stable_sampling_used = false. Otherwise, returns an empty Optional. + */ + private ListenableFuture<Optional<StableSamplingInfo>> shouldLogPerEvent(long sampleInterval) { + if (shouldSamplePerEvent(sampleInterval)) { + return immediateFuture( + Optional.of( + StableSamplingInfo.newBuilder().setStableSamplingUsed(false).build())); + } else { + return immediateFuture(Optional.absent()); + } } - } - private boolean shouldSamplePerEvent(long sampleInterval) { - if (sampleInterval == 0L) { - return false; - } else if (sampleInterval < 0L) { - LogUtil.e("Bad sample interval (negative number): %d", sampleInterval); - return false; - } else { - return isPartOfSample(random.nextLong(), sampleInterval); + private boolean shouldSamplePerEvent(long sampleInterval) { + if (sampleInterval == 0L) { + return false; + } else if (sampleInterval < 0L) { + LogUtil.e("Bad sample interval (negative number): %d", sampleInterval); + return false; + } else { + return isPartOfSample(random.nextLong(), sampleInterval); + } } - } - /** - * Returns device stable sampling. - * - * @return if the event should be sampled, returns the Void with stable_sampling_used = true and - * all other fields populated. Otherwise, returns an empty Optional. - */ - private ListenableFuture<Optional<Void>> shouldLogDeviceStable( - long sampleInterval, LoggingStateStore loggingStateStore) { - return PropagatedFluentFuture.from(loggingStateStore.getStableSamplingInfo()) - .transform( - samplingInfo -> { - boolean invalidSamplingRateUsed = ((100 % sampleInterval) != 0); - if (invalidSamplingRateUsed) { - LogUtil.e( - "Bad sample interval (1 percent cohort will not log): %d", sampleInterval); - } + /** + * Returns device stable sampling. + * + * @return if the event should be sampled, returns the StableSamplingInfo with + * stable_sampling_used = true and all other fields populated. Otherwise, returns an empty + * Optional. + */ + private ListenableFuture<Optional<StableSamplingInfo>> shouldLogDeviceStable( + long sampleInterval, LoggingStateStore loggingStateStore) { + return PropagatedFluentFuture.from(loggingStateStore.getStableSamplingInfo()) + .transform( + samplingInfo -> { + boolean invalidSamplingRateUsed = ((100 % sampleInterval) != 0); + if (invalidSamplingRateUsed) { + LogUtil.e( + "Bad sample interval (1 percent cohort will not log): %d", + sampleInterval); + } - if (!isPartOfSample(samplingInfo.getStableLogSamplingSalt(), sampleInterval)) { - return Optional.absent(); - } + if (!isPartOfSample(samplingInfo.getStableLogSamplingSalt(), + sampleInterval)) { + return Optional.absent(); + } - return Optional.absent(); - }, - directExecutor()); - } + return Optional.of( + StableSamplingInfo.newBuilder() + .setStableSamplingUsed(true) + .setStableSamplingFirstEnabledTimestampMs( + TimestampsUtil.toMillis( + samplingInfo.getLogSamplingSaltSetTimestamp())) + .setPartOfAlwaysLoggingGroup( + isPartOfSample( + samplingInfo.getStableLogSamplingSalt(), /* sampleInterval= */ + 100)) + .setInvalidSamplingRateUsed(invalidSamplingRateUsed) + .build()); + }, + directExecutor()); + } - /** - * Returns whether this device is part of the sample with the given sampling rate and random - * number. - */ - private boolean isPartOfSample(long randomNumber, long sampleInterval) { - return randomNumber % sampleInterval == 0; - } + /** + * Returns whether this device is part of the sample with the given sampling rate and random + * number. + */ + private boolean isPartOfSample(long randomNumber, long sampleInterval) { + return randomNumber % sampleInterval == 0; + } } diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/LogUtil.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/LogUtil.java index bba7ab3..bc4375e 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/LogUtil.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/LogUtil.java @@ -25,12 +25,12 @@ import java.util.Random; import javax.annotation.Nullable; /** Utility class for logging with the "MDD" tag. */ -@CanIgnoreReturnValue public class LogUtil { public static final String TAG = "MDD"; private static final Random random = new Random(); + @CanIgnoreReturnValue // pushed down from class to method; see <internal> public static int getLogPriority() { int level = Log.ASSERT; while (level > Log.VERBOSE) { @@ -42,6 +42,7 @@ public class LogUtil { return level; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> public static int v(String msg) { if (Log.isLoggable(TAG, Log.VERBOSE)) { return Log.v(TAG, msg); @@ -49,6 +50,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int v(@FormatString String format, Object obj0) { if (Log.isLoggable(TAG, Log.VERBOSE)) { @@ -58,6 +60,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int v(@FormatString String format, Object obj0, Object obj1) { if (Log.isLoggable(TAG, Log.VERBOSE)) { @@ -67,6 +70,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int v(@FormatString String format, Object... params) { if (Log.isLoggable(TAG, Log.VERBOSE)) { @@ -76,6 +80,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> public static int d(String msg) { if (Log.isLoggable(TAG, Log.DEBUG)) { return Log.d(TAG, msg); @@ -83,6 +88,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int d(@FormatString String format, Object obj0) { if (Log.isLoggable(TAG, Log.DEBUG)) { @@ -92,6 +98,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int d(@FormatString String format, Object obj0, Object obj1) { if (Log.isLoggable(TAG, Log.DEBUG)) { @@ -101,6 +108,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int d(@FormatString String format, Object... params) { if (Log.isLoggable(TAG, Log.DEBUG)) { @@ -110,6 +118,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int d(@Nullable Throwable tr, @FormatString String format, Object... params) { if (Log.isLoggable(TAG, Log.DEBUG)) { @@ -119,6 +128,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> public static int i(String msg) { if (Log.isLoggable(TAG, Log.INFO)) { return Log.i(TAG, msg); @@ -126,6 +136,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int i(@FormatString String format, Object obj0) { if (Log.isLoggable(TAG, Log.INFO)) { @@ -135,6 +146,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int i(@FormatString String format, Object obj0, Object obj1) { if (Log.isLoggable(TAG, Log.INFO)) { @@ -144,6 +156,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int i(@FormatString String format, Object... params) { if (Log.isLoggable(TAG, Log.INFO)) { @@ -153,6 +166,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> public static int e(String msg) { if (Log.isLoggable(TAG, Log.ERROR)) { return Log.e(TAG, msg); @@ -160,6 +174,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int e(@FormatString String format, Object obj0) { if (Log.isLoggable(TAG, Log.ERROR)) { @@ -169,6 +184,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int e(@FormatString String format, Object obj0, Object obj1) { if (Log.isLoggable(TAG, Log.ERROR)) { @@ -178,6 +194,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int e(@FormatString String format, Object... params) { if (Log.isLoggable(TAG, Log.ERROR)) { @@ -187,6 +204,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @SuppressLint("LogTagMismatch") public static int e(@Nullable Throwable tr, String msg) { if (Log.isLoggable(TAG, Log.ERROR)) { @@ -201,11 +219,13 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int e(@Nullable Throwable tr, @FormatString String format, Object... params) { return Log.isLoggable(TAG, Log.ERROR) ? e(tr, format(format, params)) : 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> public static int w(String msg) { if (Log.isLoggable(TAG, Log.WARN)) { return Log.w(TAG, msg); @@ -213,6 +233,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int w(@FormatString String format, Object obj0) { if (Log.isLoggable(TAG, Log.WARN)) { @@ -222,6 +243,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int w(@FormatString String format, Object obj0, Object obj1) { if (Log.isLoggable(TAG, Log.WARN)) { @@ -231,6 +253,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int w(@FormatString String format, Object... params) { if (Log.isLoggable(TAG, Log.WARN)) { @@ -240,6 +263,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @SuppressLint("LogTagMismatch") @FormatMethod public static int w(@Nullable Throwable tr, @FormatString String format, Object... params) { @@ -256,11 +280,13 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod private static String format(@FormatString String format, Object... args) { return String.format(Locale.US, format, args); } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> public static boolean shouldSampleInterval(long sampleInterval) { if (sampleInterval <= 0L) { if (sampleInterval < 0L) { diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/MddEventLogger.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/MddEventLogger.java index c2b4984..620421c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/MddEventLogger.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/MddEventLogger.java @@ -23,12 +23,21 @@ import android.content.Intent; import android.content.IntentFilter; import com.google.android.libraries.mobiledatadownload.Flags; import com.google.android.libraries.mobiledatadownload.Logger; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentFuture; import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Optional; import com.google.common.util.concurrent.AsyncCallable; -import com.google.common.util.concurrent.FluentFuture; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.ListenableFuture; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; +import com.google.mobiledatadownload.LogEnumsProto.MddDownloadResult; +import com.google.mobiledatadownload.LogProto.AndroidClientInfo; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; +import com.google.mobiledatadownload.LogProto.MddDeviceInfo; +import com.google.mobiledatadownload.LogProto.MddDownloadResultLog; +import com.google.mobiledatadownload.LogProto.MddLogData; +import com.google.mobiledatadownload.LogProto.MddStorageStats; +import com.google.mobiledatadownload.LogProto.StableSamplingInfo; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -74,75 +83,107 @@ public final class MddEventLogger implements EventLogger { } @Override - public void logEventSampled(int eventCode) {} + public void logEventSampled(MddClientEvent.Code eventCode) { + sampleAndSendLogEvent(eventCode, MddLogData.newBuilder(), flags.mddDefaultSampleInterval()); + } @Override public void logEventSampled( - int eventCode, + MddClientEvent.Code eventCode, String fileGroupName, int fileGroupVersionNumber, long buildId, String variantId) { - Void dataDownloadFileGroupStats = null; + DataDownloadFileGroupStats dataDownloadFileGroupStats = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(fileGroupName) + .setFileGroupVersionNumber(fileGroupVersionNumber) + .setBuildId(buildId) + .setVariantId(variantId) + .build(); + + sampleAndSendLogEvent( + eventCode, + MddLogData.newBuilder().setDataDownloadFileGroupStats(dataDownloadFileGroupStats), + flags.mddDefaultSampleInterval()); } @Override - public void logEventAfterSample(int eventCode, int sampleInterval) { + public void logEventAfterSample(MddClientEvent.Code eventCode, int sampleInterval) { // TODO(b/138392640): delete this method once the pds migration is complete. If it's necessary // for other use cases, we can establish a pattern where this class is still responsible for // sampling. - Void logData = null; + MddLogData.Builder logData = MddLogData.newBuilder(); processAndSendEventWithoutStableSampling(eventCode, logData, sampleInterval); } @Override - public void logMddApiCallStats(Void fileGroupDetails, Void apiCallStats) { + public void logMddApiCallStats(DataDownloadFileGroupStats fileGroupDetails, Void apiCallStats) { // TODO(b/144684763): update this to use stable sampling. Leaving it as is for now since it is // fairly high volume. long sampleInterval = flags.apiLoggingSampleInterval(); if (!LogUtil.shouldSampleInterval(sampleInterval)) { return; } - Void logData = null; - processAndSendEventWithoutStableSampling(0, logData, sampleInterval); + MddLogData.Builder logData = + MddLogData.newBuilder().setDataDownloadFileGroupStats(fileGroupDetails); + processAndSendEventWithoutStableSampling( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, logData, sampleInterval); + } + + @Override + public void logMddLibApiResultLog(Void mddLibApiResultLog) { + MddLogData.Builder logData = MddLogData.newBuilder(); + + sampleAndSendLogEvent( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, logData, flags.apiLoggingSampleInterval()); } @Override public ListenableFuture<Void> logMddFileGroupStats( AsyncCallable<List<EventLogger.FileGroupStatusWithDetails>> buildFileGroupStats) { return lazySampleAndSendLogEvent( - 0, + MddClientEvent.Code.DATA_DOWNLOAD_FILE_GROUP_STATUS, () -> PropagatedFutures.transform( buildFileGroupStats.call(), fileGroupStatusAndDetailsList -> { - List<Void> allIcingLogData = new ArrayList<>(); + List<MddLogData> allMddLogData = new ArrayList<>(); for (FileGroupStatusWithDetails fileGroupStatusAndDetails : fileGroupStatusAndDetailsList) { - allIcingLogData.add(null); + allMddLogData.add( + MddLogData.newBuilder() + .setMddFileGroupStatus(fileGroupStatusAndDetails.fileGroupStatus()) + .setDataDownloadFileGroupStats( + fileGroupStatusAndDetails.fileGroupDetails()) + .build()); } - return allIcingLogData; + return allMddLogData; }, directExecutor()), flags.groupStatsLoggingSampleInterval()); } @Override - public ListenableFuture<Void> logMddStorageStats(AsyncCallable<Void> buildStorageStats) { + public ListenableFuture<Void> logMddStorageStats( + AsyncCallable<MddStorageStats> buildStorageStats) { return lazySampleAndSendLogEvent( - 0, + MddClientEvent.Code.DATA_DOWNLOAD_STORAGE_STATS, () -> PropagatedFutures.transform( - buildStorageStats.call(), storageStats -> Arrays.asList(), directExecutor()), + buildStorageStats.call(), + storageStats -> + Arrays.asList(MddLogData.newBuilder().setMddStorageStats(storageStats).build()), + directExecutor()), flags.storageStatsLoggingSampleInterval()); } @Override public ListenableFuture<Void> logMddNetworkStats(AsyncCallable<Void> buildNetworkStats) { return lazySampleAndSendLogEvent( - 0, + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, () -> PropagatedFutures.transform( buildNetworkStats.call(), networkStats -> Arrays.asList(), directExecutor()), @@ -151,42 +192,55 @@ public final class MddEventLogger implements EventLogger { @Override public void logMddDataDownloadFileExpirationEvent(int eventCode, int count) { - Void logData = null; - sampleAndSendLogEvent(0, logData, flags.mddDefaultSampleInterval()); + MddLogData.Builder logData = MddLogData.newBuilder(); + sampleAndSendLogEvent( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, logData, flags.mddDefaultSampleInterval()); } @Override public void logMddNetworkSavings( - Void fileGroupDetails, + DataDownloadFileGroupStats fileGroupDetails, int code, long fullFileSize, long downloadedFileSize, String fileId, int deltaIndex) { - Void logData = null; + MddLogData.Builder logData = MddLogData.newBuilder(); - sampleAndSendLogEvent(0, logData, flags.mddDefaultSampleInterval()); + sampleAndSendLogEvent( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, logData, flags.mddDefaultSampleInterval()); } @Override - public void logMddQueryStats(Void fileGroupDetails) { - Void logData = null; + public void logMddQueryStats(DataDownloadFileGroupStats fileGroupDetails) { + MddLogData.Builder logData = MddLogData.newBuilder(); - sampleAndSendLogEvent(0, logData, flags.mddDefaultSampleInterval()); + sampleAndSendLogEvent( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, logData, flags.mddDefaultSampleInterval()); } @Override - public void logMddDownloadLatency(Void fileGroupDetails, Void downloadLatency) { - Void logData = null; + public void logMddDownloadLatency( + DataDownloadFileGroupStats fileGroupDetails, Void downloadLatency) { + MddLogData.Builder logData = + MddLogData.newBuilder().setDataDownloadFileGroupStats(fileGroupDetails); - sampleAndSendLogEvent(0, logData, flags.mddDefaultSampleInterval()); + sampleAndSendLogEvent( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, logData, flags.mddDefaultSampleInterval()); } @Override - public void logMddDownloadResult(int code, Void fileGroupDetails) { - Void logData = null; - - sampleAndSendLogEvent(0, logData, flags.mddDefaultSampleInterval()); + public void logMddDownloadResult( + MddDownloadResult.Code code, DataDownloadFileGroupStats fileGroupDetails) { + MddLogData.Builder logData = + MddLogData.newBuilder() + .setMddDownloadResultLog( + MddDownloadResultLog.newBuilder() + .setResult(code) + .setDataDownloadFileGroupStats(fileGroupDetails)); + + sampleAndSendLogEvent( + MddClientEvent.Code.DATA_DOWNLOAD_RESULT_LOG, logData, flags.mddDefaultSampleInterval()); } @Override @@ -196,15 +250,27 @@ public final class MddEventLogger implements EventLogger { if (!LogUtil.shouldSampleInterval(sampleInterval)) { return; } - Void logData = null; - processAndSendEventWithoutStableSampling(0, logData, sampleInterval); + MddLogData.Builder logData = MddLogData.newBuilder(); + processAndSendEventWithoutStableSampling( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, logData, sampleInterval); } @Override - public void logMddUsageEvent(Void fileGroupDetails, Void usageEventLog) { - Void logData = null; + public void logMddUsageEvent(DataDownloadFileGroupStats fileGroupDetails, Void usageEventLog) { + MddLogData.Builder logData = + MddLogData.newBuilder().setDataDownloadFileGroupStats(fileGroupDetails); - sampleAndSendLogEvent(0, logData, flags.mddDefaultSampleInterval()); + sampleAndSendLogEvent( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, logData, flags.mddDefaultSampleInterval()); + } + + @Override + public void logNewConfigReceived( + DataDownloadFileGroupStats fileGroupDetails, Void newConfigReceivedInfo) { + MddLogData.Builder logData = + MddLogData.newBuilder().setDataDownloadFileGroupStats(fileGroupDetails); + sampleAndSendLogEvent( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, logData, flags.mddDefaultSampleInterval()); } /** @@ -213,7 +279,9 @@ public final class MddEventLogger implements EventLogger { * constructs the log event lazy. This is useful if constructing the log event is expensive. */ private ListenableFuture<Void> lazySampleAndSendLogEvent( - int eventCode, AsyncCallable<List<Void>> buildStats, int sampleInterval) { + MddClientEvent.Code eventCode, + AsyncCallable<List<MddLogData>> buildStats, + int sampleInterval) { return PropagatedFutures.transformAsync( logSampler.shouldLog(sampleInterval, loggingStateStore), samplingInfoOptional -> { @@ -221,13 +289,16 @@ public final class MddEventLogger implements EventLogger { return immediateVoidFuture(); } - return FluentFuture.from(buildStats.call()) + return PropagatedFluentFuture.from(buildStats.call()) .transform( icingLogDataList -> { if (icingLogDataList != null) { - for (Void icingLogData : icingLogDataList) { + for (MddLogData icingLogData : icingLogDataList) { processAndSendEvent( - eventCode, null, sampleInterval, samplingInfoOptional.get()); + eventCode, + icingLogData.toBuilder(), + sampleInterval, + samplingInfoOptional.get()); } } return null; @@ -237,12 +308,15 @@ public final class MddEventLogger implements EventLogger { directExecutor()); } - private void sampleAndSendLogEvent(int eventCode, Void logData, long sampleInterval) { + private void sampleAndSendLogEvent( + MddClientEvent.Code eventCode, MddLogData.Builder logData, long sampleInterval) { + // NOTE: When using a single-threaded executor, logging may be delayed since other + // work will come before the log sampler check. PropagatedFutures.addCallback( logSampler.shouldLog(sampleInterval, loggingStateStore), - new FutureCallback<Optional<Void>>() { + new FutureCallback<Optional<StableSamplingInfo>>() { @Override - public void onSuccess(Optional<Void> stableSamplingInfo) { + public void onSuccess(Optional<StableSamplingInfo> stableSamplingInfo) { if (stableSamplingInfo.isPresent()) { processAndSendEvent(eventCode, logData, sampleInterval, stableSamplingInfo.get()); } @@ -258,13 +332,35 @@ public final class MddEventLogger implements EventLogger { /** Adds all transforms common to all logs and sends the event to Logger. */ private void processAndSendEventWithoutStableSampling( - int eventCode, Void logData, long sampleInterval) { - processAndSendEvent(eventCode, logData, sampleInterval, null); + MddClientEvent.Code eventCode, MddLogData.Builder logData, long sampleInterval) { + processAndSendEvent( + eventCode, + logData, + sampleInterval, + StableSamplingInfo.newBuilder().setStableSamplingUsed(false).build()); } /** Adds all transforms common to all logs and sends the event to Logger. */ private void processAndSendEvent( - int eventCode, Void logData, long sampleInterval, Void stableSamplingInfo) {} + MddClientEvent.Code eventCode, + MddLogData.Builder logData, + long sampleInterval, + StableSamplingInfo stableSamplingInfo) { + if (eventCode.equals(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED)) { + LogUtil.e("%s: unspecified code used, skipping event log", TAG); + // return early for unspecified codes. + return; + } + logData + .setSamplingInterval(sampleInterval) + .setDeviceInfo(MddDeviceInfo.newBuilder().setDeviceStorageLow(isDeviceStorageLow(context))) + .setAndroidClientInfo( + AndroidClientInfo.newBuilder() + .setHostPackageName(hostPackageName) + .setModuleVersion(moduleVersion)) + .setStableSamplingInfo(stableSamplingInfo); + logger.log(logData.build(), eventCode.getNumber()); + } /** Returns whether the device is in low storage state. */ private static boolean isDeviceStorageLow(Context context) { diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/NoOpEventLogger.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/NoOpEventLogger.java index 5bf6ebc..2f043f5 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/NoOpEventLogger.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/NoOpEventLogger.java @@ -19,24 +19,28 @@ import static com.google.common.util.concurrent.Futures.immediateVoidFuture; import com.google.common.util.concurrent.AsyncCallable; import com.google.common.util.concurrent.ListenableFuture; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; +import com.google.mobiledatadownload.LogEnumsProto.MddDownloadResult; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; +import com.google.mobiledatadownload.LogProto.MddStorageStats; import java.util.List; /** No-Op EventLogger implementation. */ public final class NoOpEventLogger implements EventLogger { @Override - public void logEventSampled(int eventCode) {} + public void logEventSampled(MddClientEvent.Code eventCode) {} @Override public void logEventSampled( - int eventCode, + MddClientEvent.Code eventCode, String fileGroupName, int fileGroupVersionNumber, long buildId, String variantId) {} @Override - public void logEventAfterSample(int eventCode, int sampleInterval) {} + public void logEventAfterSample(MddClientEvent.Code eventCode, int sampleInterval) {} @Override public ListenableFuture<Void> logMddFileGroupStats( @@ -45,10 +49,14 @@ public final class NoOpEventLogger implements EventLogger { } @Override - public void logMddApiCallStats(Void fileGroupDetails, Void apiCallStats) {} + public void logMddApiCallStats(DataDownloadFileGroupStats fileGroupDetails, Void apiCallStats) {} @Override - public ListenableFuture<Void> logMddStorageStats(AsyncCallable<Void> buildMddStorageStats) { + public void logMddLibApiResultLog(Void mddLibApiResultLog) {} + + @Override + public ListenableFuture<Void> logMddStorageStats( + AsyncCallable<MddStorageStats> buildMddStorageStats) { return immediateVoidFuture(); } @@ -62,7 +70,7 @@ public final class NoOpEventLogger implements EventLogger { @Override public void logMddNetworkSavings( - Void fileGroupDetails, + DataDownloadFileGroupStats fileGroupDetails, int code, long fullFileSize, long downloadedFileSize, @@ -70,17 +78,23 @@ public final class NoOpEventLogger implements EventLogger { int deltaIndex) {} @Override - public void logMddDownloadResult(int code, Void fileGroupDetails) {} + public void logMddDownloadResult( + MddDownloadResult.Code code, DataDownloadFileGroupStats fileGroupDetails) {} @Override - public void logMddQueryStats(Void fileGroupDetails) {} + public void logMddQueryStats(DataDownloadFileGroupStats fileGroupDetails) {} @Override public void logMddAndroidSharingLog(Void event) {} @Override - public void logMddDownloadLatency(Void fileGroupStats, Void downloadLatency) {} + public void logMddDownloadLatency( + DataDownloadFileGroupStats fileGroupStats, Void downloadLatency) {} + + @Override + public void logMddUsageEvent(DataDownloadFileGroupStats fileGroupDetails, Void usageEventLog) {} @Override - public void logMddUsageEvent(Void fileGroupDetails, Void usageEventLog) {} + public void logNewConfigReceived( + DataDownloadFileGroupStats fileGroupDetails, Void newConfigReceivedInfo) {} } diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/SharedPreferencesLoggingState.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/SharedPreferencesLoggingState.java new file mode 100644 index 0000000..7fd90ef --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/SharedPreferencesLoggingState.java @@ -0,0 +1,350 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.internal.logging; + +import static com.google.android.libraries.mobiledatadownload.internal.MddConstants.SPLIT_CHAR; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +import android.content.Context; +import android.content.SharedPreferences; + +import androidx.annotation.VisibleForTesting; + +import com.google.android.libraries.mobiledatadownload.TimeSource; +import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupsMetadataUtil; +import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupsMetadataUtil.GroupKeyDeserializationException; +import com.google.android.libraries.mobiledatadownload.internal.util.SharedPreferencesUtil; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedExecutionSequencer; +import com.google.common.base.Optional; +import com.google.common.base.Splitter; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import com.google.common.primitives.Ints; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.mobiledatadownload.internal.MetadataProto.FileGroupLoggingState; +import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; +import com.google.mobiledatadownload.internal.MetadataProto.SamplingInfo; +import com.google.protobuf.Timestamp; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.GregorianCalendar; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.TimeZone; +import java.util.concurrent.Executor; + +/** LoggingStateStore that uses SharedPreferences for storage. */ +public final class SharedPreferencesLoggingState implements LoggingStateStore { + + private static final String SHARED_PREFS_NAME = "LoggingState"; + + private static final String LAST_MAINTENANCE_RUN_SECS_KEY = "last_maintenance_secs"; + + @VisibleForTesting + static final String SALT_KEY = "stable_log_sampling_salt"; + private static final String SALT_TIMESTAMP_MILLIS_KEY = + "log_sampling_salt_set_timestamp_millis"; + + private final Supplier<SharedPreferences> sharedPrefs; + private final Executor backgroundExecutor; + private final TimeSource timeSource; + private final Random random; + + // Serialize access to SharedPref keys to avoid clobbering. + private final PropagatedExecutionSequencer futureSerializer = + PropagatedExecutionSequencer.create(); + + /** + * Constructs a new instance. + * + * @param sharedPrefs may be called multiple times, so memoization is recommended. The returned + * instance must be exclusive to {@link SharedPreferencesLoggingState} since + * {@link #clear} + * may clear the data at any time. + */ + public static SharedPreferencesLoggingState create( + Supplier<SharedPreferences> sharedPrefs, + TimeSource timeSource, + Executor backgroundExecutor, + Random random) { + return new SharedPreferencesLoggingState(sharedPrefs, timeSource, backgroundExecutor, + random); + } + + /** Constructs a new instance. */ + public static SharedPreferencesLoggingState createFromContext( + Context context, + Optional<String> instanceIdOptional, + TimeSource timeSource, + Executor backgroundExecutor, + Random random) { + // Avoid calling getSharedPreferences on the main thread. + Supplier<SharedPreferences> sharedPrefs = + Suppliers.memoize( + () -> + SharedPreferencesUtil.getSharedPreferences( + context, SHARED_PREFS_NAME, instanceIdOptional)); + return new SharedPreferencesLoggingState(sharedPrefs, timeSource, backgroundExecutor, + random); + } + + private SharedPreferencesLoggingState( + Supplier<SharedPreferences> sharedPrefs, + TimeSource timeSource, + Executor backgroundExecutor, + Random random) { + this.sharedPrefs = sharedPrefs; + this.timeSource = timeSource; + this.backgroundExecutor = backgroundExecutor; + this.random = random; + } + + /** Data fields for each Entry persisted in SharedPreferences. */ + private enum Key { + CELLULAR_USAGE("cu"), + WIFI_USAGE("wu"); + + final String sharedPrefsSuffix; + + Key(String sharedPrefsSuffix) { + this.sharedPrefsSuffix = sharedPrefsSuffix; + } + } + + /** Bridge between FileGroupLoggingState and its SharedPreferences representation. */ + private static final class Entry { + + final GroupKey groupKey; + final long buildId; + final int fileGroupVersionNumber; + + /** Prefix used in SharedPreference keys. */ + final String spKeyPrefix; + + static Entry fromLoggingState(FileGroupLoggingState loggingState) { + return new Entry( + /* groupKey= */ loggingState.getGroupKey(), + /* buildId= */ loggingState.getBuildId(), + /* fileGroupVersionNumber= */ loggingState.getFileGroupVersionNumber()); + } + + /** + * @throws IllegalArgumentException if the key can't be parsed + */ + static Entry fromSpKey(String spKey) { + List<String> parts = Splitter.on(SPLIT_CHAR).splitToList(spKey); + try { + return new Entry( + /* groupKey= */ FileGroupsMetadataUtil.deserializeGroupKey(parts.get(0)), + /* buildId= */ Long.parseLong(parts.get(1)), + /* fileGroupVersionNumber= */ Integer.parseInt(parts.get(2))); + } catch (GroupKeyDeserializationException | ArrayIndexOutOfBoundsException e) { + throw new IllegalArgumentException("Failed to parse SharedPrefs key: " + spKey, e); + } + } + + private Entry(GroupKey groupKey, long buildId, int fileGroupVersionNumber) { + this.groupKey = groupKey; + this.buildId = buildId; + this.fileGroupVersionNumber = fileGroupVersionNumber; + this.spKeyPrefix = + FileGroupsMetadataUtil.getSerializedGroupKey(groupKey) + + SPLIT_CHAR + + buildId + + SPLIT_CHAR + + fileGroupVersionNumber; + } + + String getSharedPrefsKey(Key key) { + return spKeyPrefix + SPLIT_CHAR + key.sharedPrefsSuffix; + } + } + + @Override + public ListenableFuture<Optional<Integer>> getAndResetDaysSinceLastMaintenance() { + return futureSerializer.submit( + () -> { + long currentTimestamp = timeSource.currentTimeMillis(); + + Optional<Integer> daysSinceLastMaintenance; + boolean hasEverDoneMaintenance = + sharedPrefs.get().contains(LAST_MAINTENANCE_RUN_SECS_KEY); + if (hasEverDoneMaintenance) { + long persistedTimestamp = sharedPrefs.get().getLong( + LAST_MAINTENANCE_RUN_SECS_KEY, 0); + long currentStartOfDay = truncateTimestampToStartOfDay(currentTimestamp); + long previousStartOfDay = truncateTimestampToStartOfDay(persistedTimestamp); + // Note: ignore MillisTo_Days java optional suggestion because Duration + // is api + // 26+. + daysSinceLastMaintenance = + Optional.of( + Ints.saturatedCast( + MILLISECONDS.toDays( + currentStartOfDay - previousStartOfDay))); + } else { + daysSinceLastMaintenance = Optional.absent(); + } + + SharedPreferences.Editor editor = sharedPrefs.get().edit(); + editor.putLong(LAST_MAINTENANCE_RUN_SECS_KEY, currentTimestamp); + commitOrThrow(editor); + + return daysSinceLastMaintenance; + }, + backgroundExecutor); + } + + @Override + public ListenableFuture<Void> incrementDataUsage(FileGroupLoggingState dataUsageIncrements) { + return futureSerializer.submit( + () -> { + Entry entry = Entry.fromLoggingState(dataUsageIncrements); + + long currentCellarUsage = + sharedPrefs.get().getLong(entry.getSharedPrefsKey(Key.CELLULAR_USAGE), + 0); + long currentWifiUsage = + sharedPrefs.get().getLong(entry.getSharedPrefsKey(Key.WIFI_USAGE), 0); + long updatedCellarUsage = + currentCellarUsage + dataUsageIncrements.getCellularUsage(); + long updatedWifiUsage = currentWifiUsage + dataUsageIncrements.getWifiUsage(); + + SharedPreferences.Editor editor = sharedPrefs.get().edit(); + editor.putLong(entry.getSharedPrefsKey(Key.CELLULAR_USAGE), updatedCellarUsage); + editor.putLong(entry.getSharedPrefsKey(Key.WIFI_USAGE), updatedWifiUsage); + + return commitOrThrow(editor); + }, + backgroundExecutor); + } + + @Override + public ListenableFuture<List<FileGroupLoggingState>> getAndResetAllDataUsage() { + return futureSerializer.submit( + () -> { + List<FileGroupLoggingState> allLoggingStates = new ArrayList<>(); + Set<String> allLoggingStateKeys = new HashSet<>(); + SharedPreferences.Editor editor = sharedPrefs.get().edit(); + + for (String key : sharedPrefs.get().getAll().keySet()) { + Entry entry; + try { + entry = Entry.fromSpKey(key); + } catch (IllegalArgumentException e) { + continue; // This isn't a LoggingState entry + } + if (allLoggingStateKeys.contains(entry.spKeyPrefix)) { + continue; + } + allLoggingStateKeys.add(entry.spKeyPrefix); + + FileGroupLoggingState loggingState = + FileGroupLoggingState.newBuilder() + .setGroupKey(entry.groupKey) + .setBuildId(entry.buildId) + .setFileGroupVersionNumber(entry.fileGroupVersionNumber) + .setCellularUsage( + sharedPrefs.get().getLong( + entry.getSharedPrefsKey(Key.CELLULAR_USAGE), + 0)) + .setWifiUsage( + sharedPrefs.get().getLong( + entry.getSharedPrefsKey(Key.WIFI_USAGE), 0)) + .build(); + allLoggingStates.add(loggingState); + + editor.remove(entry.getSharedPrefsKey(Key.CELLULAR_USAGE)); + editor.remove(entry.getSharedPrefsKey(Key.WIFI_USAGE)); + } + commitOrThrow(editor); + + return allLoggingStates; + }, + backgroundExecutor); + } + + @Override + public ListenableFuture<Void> clear() { + return futureSerializer.submit( + () -> { + SharedPreferences.Editor editor = sharedPrefs.get().edit(); + editor.clear(); + return commitOrThrow(editor); + }, + backgroundExecutor); + } + + @Override + public ListenableFuture<SamplingInfo> getStableSamplingInfo() { + return futureSerializer.submit( + () -> { + long salt; + long persistedTimestampMillis; + + boolean hasCreatedSalt = sharedPrefs.get().contains(SALT_KEY); + if (hasCreatedSalt) { + salt = sharedPrefs.get().getLong(SALT_KEY, 0); + persistedTimestampMillis = sharedPrefs.get().getLong( + SALT_TIMESTAMP_MILLIS_KEY, 0); + } else { + salt = random.nextLong(); + persistedTimestampMillis = timeSource.currentTimeMillis(); + + SharedPreferences.Editor editor = sharedPrefs.get().edit(); + editor.putLong(SALT_KEY, salt); + editor.putLong(SALT_TIMESTAMP_MILLIS_KEY, persistedTimestampMillis); + commitOrThrow(editor); + } + + Timestamp timestamp = TimestampsUtil.fromMillis(persistedTimestampMillis); + return SamplingInfo.newBuilder() + .setStableLogSamplingSalt(salt) + .setLogSamplingSaltSetTimestamp(timestamp) + .build(); + }, + backgroundExecutor); + } + + // Use UTC time zone here so we don't have to worry about time zone change or daylight savings. + private static final TimeZone UTC_TIMEZONE = TimeZone.getTimeZone("UTC"); + + // TODO(b/237533403): extract as shareable code with ProtoDataStoreLoggingState + private static long truncateTimestampToStartOfDay(long timestampMillis) { + // We use the regular java.util.Calendar classes here since neither Joda time nor java.time is + // supported across all client apps. + Calendar cal = new GregorianCalendar(UTC_TIMEZONE); + cal.setTimeInMillis(timestampMillis); + cal.set(Calendar.HOUR_OF_DAY, 0); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + return cal.getTimeInMillis(); + } + + /** Calls {@code editor.commit()} and returns void, or throws IOException if the commit failed. */ + private static Void commitOrThrow(SharedPreferences.Editor editor) throws IOException { + if (!editor.commit()) { + throw new IOException("Failed to commit"); + } + return null; + } +} diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/StorageLogger.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/StorageLogger.java index 5307941..d707f42 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/StorageLogger.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/StorageLogger.java @@ -16,12 +16,14 @@ package com.google.android.libraries.mobiledatadownload.internal.logging; import static com.google.android.libraries.mobiledatadownload.internal.MddConstants.SPLIT_CHAR; +import static com.google.common.util.concurrent.Futures.immediateFuture; import android.content.Context; -import android.util.Pair; +import android.net.Uri; import com.google.android.libraries.mobiledatadownload.SilentFeedback; import com.google.android.libraries.mobiledatadownload.annotations.InstanceId; import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; +import com.google.android.libraries.mobiledatadownload.file.openers.RecursiveSizeOpener; import com.google.android.libraries.mobiledatadownload.internal.ApplicationContext; import com.google.android.libraries.mobiledatadownload.internal.FileGroupsMetadata; import com.google.android.libraries.mobiledatadownload.internal.MddConstants; @@ -29,16 +31,17 @@ import com.google.android.libraries.mobiledatadownload.internal.SharedFileManage import com.google.android.libraries.mobiledatadownload.internal.SharedFileMissingException; import com.google.android.libraries.mobiledatadownload.internal.SharedFilesMetadata; import com.google.android.libraries.mobiledatadownload.internal.annotations.SequentialControlExecutor; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup; +import com.google.android.libraries.mobiledatadownload.internal.util.DirectoryUtil; import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupUtil; import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentFuture; import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; -import com.google.auto.value.AutoValue; import com.google.common.base.Optional; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import com.google.common.util.concurrent.FluentFuture; -import com.google.common.util.concurrent.Futures; +import com.google.common.base.Splitter; import com.google.common.util.concurrent.ListenableFuture; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; +import com.google.mobiledatadownload.LogProto.MddStorageStats; import com.google.mobiledatadownload.internal.MetadataProto.DataFile; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; @@ -114,7 +117,7 @@ public class StorageLogger { private static GroupKey createGroupKey(DataFileGroupInternal fileGroup) { GroupKey.Builder groupKey = GroupKey.newBuilder().setGroupName(fileGroup.getGroupName()); - if (Strings.isNullOrEmpty(fileGroup.getOwnerPackage())) { + if (fileGroup.getOwnerPackage().isEmpty()) { groupKey.setOwnerPackage(MddConstants.GMS_PACKAGE); } else { groupKey.setOwnerPackage(fileGroup.getOwnerPackage()); @@ -124,10 +127,10 @@ public class StorageLogger { } public ListenableFuture<Void> logStorageStats(int daysSinceLastLog) { - return eventLogger.logMddStorageStats(() -> buildStorageStatsIcingLogData(daysSinceLastLog)); + return eventLogger.logMddStorageStats(() -> buildStorageStatsLogData(daysSinceLastLog)); } - private ListenableFuture<Void> buildStorageStatsIcingLogData(int daysSinceLastLog) { + private ListenableFuture<MddStorageStats> buildStorageStatsLogData(int daysSinceLastLog) { return PropagatedFluentFuture.from(fileGroupsMetadata.getAllFreshGroups()) .transformAsync( allGroups -> @@ -139,21 +142,19 @@ public class StorageLogger { sequentialControlExecutor); } - private ListenableFuture<Void> buildStorageStatsInternal( - List<Pair<GroupKey, DataFileGroupInternal>> allKeysAndGroupPairs, + private ListenableFuture<MddStorageStats> buildStorageStatsInternal( + List<GroupKeyAndGroup> allKeysAndGroupPairs, List<DataFileGroupInternal> staleGroups, int daysSinceLastLog) { - List<GroupKeyAndDataFileGroupInternal> allKeysAndGroups = new ArrayList<>(); - for (Pair<GroupKey, DataFileGroupInternal> groupKeyAndGroup : allKeysAndGroupPairs) { - allKeysAndGroups.add( - GroupKeyAndDataFileGroupInternal.create(groupKeyAndGroup.first, groupKeyAndGroup.second)); + List<GroupKeyAndGroup> allKeysAndGroups = new ArrayList<>(); + for (GroupKeyAndGroup groupKeyAndGroup : allKeysAndGroupPairs) { + allKeysAndGroups.add(groupKeyAndGroup); } // Adding staleGroups to allGroups. for (DataFileGroupInternal fileGroup : staleGroups) { - allKeysAndGroups.add( - GroupKeyAndDataFileGroupInternal.create(createGroupKey(fileGroup), fileGroup)); + allKeysAndGroups.add(GroupKeyAndGroup.create(createGroupKey(fileGroup), fileGroup)); } Map<String, GroupStorage> groupKeyToGroupStorage = new HashMap<>(); @@ -168,7 +169,7 @@ public class StorageLogger { AtomicLong totalMddBytesUsed = new AtomicLong(0L); List<ListenableFuture<Void>> futures = new ArrayList<>(); - for (GroupKeyAndDataFileGroupInternal groupKeyAndGroup : allKeysAndGroups) { + for (GroupKeyAndGroup groupKeyAndGroup : allKeysAndGroups) { Set<NewFileKey> fileKeys = safeGetFileKeys( @@ -187,20 +188,20 @@ public class StorageLogger { getGroupWithOwnerPackageKey(groupKeyAndGroup.groupKey())); downloadedGroupKeyToDataFileGroup.put( getGroupWithOwnerPackageKey(groupKeyAndGroup.groupKey()), - groupKeyAndGroup.dataFileGroupInternal()); + groupKeyAndGroup.dataFileGroup()); } // Variables captured by lambdas must be effectively final. Set<NewFileKey> downloadedFileKeys = downloadedFileKeysInit; - int totalFileCount = groupKeyAndGroup.dataFileGroupInternal().getFileCount(); - for (DataFile dataFile : groupKeyAndGroup.dataFileGroupInternal().getFileList()) { + int totalFileCount = groupKeyAndGroup.dataFileGroup().getFileCount(); + for (DataFile dataFile : groupKeyAndGroup.dataFileGroup().getFileList()) { boolean isInlineFile = FileGroupUtil.isInlineFile(dataFile); NewFileKey fileKey = SharedFilesMetadata.createKeyFromDataFile( - dataFile, groupKeyAndGroup.dataFileGroupInternal().getAllowedReadersEnum()); + dataFile, groupKeyAndGroup.dataFileGroup().getAllowedReadersEnum()); futures.add( - Futures.transform( + PropagatedFutures.transform( computeFileSize(fileKey), fileSize -> { if (!allFileKeys.contains(fileKey)) { @@ -240,11 +241,65 @@ public class StorageLogger { groupStorage.totalFileCount = totalFileCount; } - return Futures.whenAllComplete(futures) + return PropagatedFutures.whenAllComplete(futures) .call( () -> { - Void storageStatsBuilder = null; - return storageStatsBuilder; + MddStorageStats.Builder storageStatsBuilder = MddStorageStats.newBuilder(); + for (String groupName : groupKeyToGroupStorage.keySet()) { + GroupStorage groupStorage = groupKeyToGroupStorage.get(groupName); + List<String> groupNameAndOwnerPackage = + Splitter.on(SPLIT_CHAR).splitToList(groupName); + + DataDownloadFileGroupStats.Builder fileGroupDetailsBuilder = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(groupNameAndOwnerPackage.get(0)) + .setOwnerPackage(groupNameAndOwnerPackage.get(1)) + .setFileCount(groupStorage.totalFileCount) + .setInlineFileCount(groupStorage.totalInlineFileCount); + + DataFileGroupInternal dataFileGroup = + downloadedGroupKeyToDataFileGroup.get(groupName); + + if (dataFileGroup == null) { + fileGroupDetailsBuilder.setFileGroupVersionNumber(-1); + } else { + fileGroupDetailsBuilder + .setFileGroupVersionNumber(dataFileGroup.getFileGroupVersionNumber()) + .setBuildId(dataFileGroup.getBuildId()) + .setVariantId(dataFileGroup.getVariantId()); + } + + storageStatsBuilder.addDataDownloadFileGroupStats(fileGroupDetailsBuilder.build()); + + storageStatsBuilder.addTotalBytesUsed(groupStorage.totalBytesUsed); + storageStatsBuilder.addTotalInlineBytesUsed(groupStorage.totalInlineBytesUsed); + storageStatsBuilder.addDownloadedGroupBytesUsed( + groupStorage.downloadedGroupBytesUsed); + storageStatsBuilder.addDownloadedGroupInlineBytesUsed( + groupStorage.downloadedGroupInlineBytesUsed); + } + + storageStatsBuilder.setTotalMddBytesUsed(totalMddBytesUsed.get()); + + long mddDirectoryBytesUsed = 0; + try { + Uri uri = DirectoryUtil.getBaseDownloadDirectory(context, instanceId); + if (fileStorage.exists(uri)) { + mddDirectoryBytesUsed = fileStorage.open(uri, RecursiveSizeOpener.create()); + } + } catch (IOException e) { + mddDirectoryBytesUsed = 0; + LogUtil.e( + e, "%s: Failed to call Mobstore to compute MDD Directory bytes used!", TAG); + silentFeedback.send( + e, "Failed to call Mobstore to compute MDD Directory bytes used!"); + } + + storageStatsBuilder + .setTotalMddDirectoryBytesUsed(mddDirectoryBytesUsed) + .setDaysSinceLastLog(daysSinceLastLog); + + return storageStatsBuilder.build(); }, sequentialControlExecutor); } @@ -277,11 +332,9 @@ public class StorageLogger { } private ListenableFuture<Long> computeFileSize(NewFileKey newFileKey) { - return FluentFuture.from(sharedFileManager.getOnDeviceUri(newFileKey)) + return PropagatedFluentFuture.from(sharedFileManager.getOnDeviceUri(newFileKey)) .catchingAsync( - SharedFileMissingException.class, - e -> Futures.immediateFuture(null), - sequentialControlExecutor) + SharedFileMissingException.class, e -> immediateFuture(null), sequentialControlExecutor) .transform( fileUri -> { if (fileUri != null) { @@ -295,17 +348,4 @@ public class StorageLogger { }, sequentialControlExecutor); } - - @AutoValue - abstract static class GroupKeyAndDataFileGroupInternal { - static GroupKeyAndDataFileGroupInternal create( - GroupKey groupKey, DataFileGroupInternal dataFileGroupInternal) { - return new AutoValue_StorageLogger_GroupKeyAndDataFileGroupInternal( - groupKey, dataFileGroupInternal); - } - - abstract GroupKey groupKey(); - - abstract DataFileGroupInternal dataFileGroupInternal(); - } } diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/TimestampsUtil.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/TimestampsUtil.java new file mode 100644 index 0000000..e0f2205 --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/TimestampsUtil.java @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.libraries.mobiledatadownload.internal.logging; + +import static com.google.common.math.LongMath.checkedAdd; +import static com.google.common.math.LongMath.checkedMultiply; +import static com.google.common.math.LongMath.checkedSubtract; + +import com.google.protobuf.Timestamp; + +/** + * Utilities to help create/manipulate {@code protobuf/timestamp.proto}. + */ +public class TimestampsUtil { + + // Timestamp for "0001-01-01T00:00:00Z" + static final long TIMESTAMP_SECONDS_MIN = -62135596800L; + + // Timestamp for "9999-12-31T23:59:59Z" + static final long TIMESTAMP_SECONDS_MAX = 253402300799L; + + static final int NANOS_PER_SECOND = 1000000000; + static final int NANOS_PER_MILLISECOND = 1000000; + static final int NANOS_PER_MICROSECOND = 1000; + static final int MILLIS_PER_SECOND = 1000; + static final int MICROS_PER_SECOND = 1000000; + + @SuppressWarnings("GoodTime") // this is a legacy conversion API + public static long toMillis(Timestamp timestamp) { + checkValid(timestamp); + return checkedAdd( + checkedMultiply(timestamp.getSeconds(), MILLIS_PER_SECOND), + timestamp.getNanos() / NANOS_PER_MILLISECOND); + } + + + /** Create a Timestamp from the number of milliseconds elapsed from the epoch. */ + @SuppressWarnings("GoodTime") // this is a legacy conversion API + public static Timestamp fromMillis(long milliseconds) { + return normalizedTimestamp( + milliseconds / MILLIS_PER_SECOND, + (int) (milliseconds % MILLIS_PER_SECOND * NANOS_PER_MILLISECOND)); + } + + public static Timestamp checkValid(Timestamp timestamp) { + long seconds = timestamp.getSeconds(); + int nanos = timestamp.getNanos(); + if (!isValid(seconds, nanos)) { + throw new IllegalArgumentException( + String.format( + "Timestamp is not valid. See proto definition for valid values. " + + "Seconds (%s) must be in range [-62,135,596,800, +253,402," + + "300,799]. " + + "Nanos (%s) must be in range [0, +999,999,999].", + seconds, nanos)); + } + return timestamp; + } + + /** + * Returns true if the given number of seconds and nanos is a valid {@link Timestamp}. The + * {@code + * seconds} value must be in the range [-62,135,596,800, +253,402,300,799] (i.e., between + * 0001-01-01T00:00:00Z and 9999-12-31T23:59:59Z). The {@code nanos} value must be in the range + * [0, +999,999,999]. + * + * <p><b>Note:</b> Negative second values with fractional seconds must still have non-negative + * nanos values that count forward in time. + */ + @SuppressWarnings("GoodTime") // this is a legacy conversion API + public static boolean isValid(long seconds, int nanos) { + if (seconds < TIMESTAMP_SECONDS_MIN || seconds > TIMESTAMP_SECONDS_MAX) { + return false; + } + if (nanos < 0 || nanos >= NANOS_PER_SECOND) { + return false; + } + return true; + } + + static Timestamp normalizedTimestamp(long seconds, int nanos) { + if (nanos <= -NANOS_PER_SECOND || nanos >= NANOS_PER_SECOND) { + seconds = checkedAdd(seconds, nanos / NANOS_PER_SECOND); + nanos = (int) (nanos % NANOS_PER_SECOND); + } + if (nanos < 0) { + nanos = + (int) + (nanos + + NANOS_PER_SECOND); // no overflow since nanos is negative + // (and we're adding) + seconds = checkedSubtract(seconds, 1); + } + Timestamp timestamp = Timestamp.newBuilder().setSeconds(seconds).setNanos(nanos).build(); + return checkValid(timestamp); + } +} diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/testing/BUILD b/java/com/google/android/libraries/mobiledatadownload/internal/logging/testing/BUILD index 9bf9510..1a20ff9 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/testing/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/testing/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -26,6 +27,8 @@ android_library( srcs = ["FakeEventLogger.java"], deps = [ "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "@com_google_guava_guava", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/testing/FakeEventLogger.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/testing/FakeEventLogger.java index e2dba29..ea5134c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/testing/FakeEventLogger.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/testing/FakeEventLogger.java @@ -22,23 +22,32 @@ import com.google.android.libraries.mobiledatadownload.internal.logging.EventLog import com.google.common.collect.ArrayListMultimap; import com.google.common.util.concurrent.AsyncCallable; import com.google.common.util.concurrent.ListenableFuture; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; +import com.google.mobiledatadownload.LogEnumsProto.MddDownloadResult; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; +import com.google.mobiledatadownload.LogProto.MddStorageStats; import java.util.ArrayList; import java.util.List; /** Fake implementation of {@link EventLogger} for use in tests. */ public final class FakeEventLogger implements EventLogger { - private final ArrayList<Integer> loggedCodes = new ArrayList<>(); - private final ArrayListMultimap<Void, Void> loggedLatencies = ArrayListMultimap.create(); + private final ArrayList<MddClientEvent.Code> loggedCodes = new ArrayList<>(); + private final ArrayListMultimap<DataDownloadFileGroupStats, Void> loggedLatencies = + ArrayListMultimap.create(); + private final ArrayListMultimap<DataDownloadFileGroupStats, Void> loggedNewConfigReceived = + ArrayListMultimap.create(); + private final List<Void> loggedMddLibApiResultLog = new ArrayList<>(); + private final ArrayList<DataDownloadFileGroupStats> loggedMddQueryStats = new ArrayList<>(); @Override - public void logEventSampled(int eventCode) { + public void logEventSampled(MddClientEvent.Code eventCode) { loggedCodes.add(eventCode); } @Override public void logEventSampled( - int eventCode, + MddClientEvent.Code eventCode, String fileGroupName, int fileGroupVersionNumber, long buildId, @@ -47,7 +56,7 @@ public final class FakeEventLogger implements EventLogger { } @Override - public void logEventAfterSample(int eventCode, int sampleInterval) { + public void logEventAfterSample(MddClientEvent.Code eventCode, int sampleInterval) { loggedCodes.add(eventCode); } @@ -59,12 +68,22 @@ public final class FakeEventLogger implements EventLogger { } @Override - public void logMddApiCallStats(Void fileGroupDetails, Void apiCallStats) { + public void logMddApiCallStats(DataDownloadFileGroupStats fileGroupDetails, Void apiCallStats) { throw new UnsupportedOperationException("This method is not implemented in the fake yet."); } @Override - public ListenableFuture<Void> logMddStorageStats(AsyncCallable<Void> buildMddStorageStats) { + public void logMddLibApiResultLog(Void mddLibApiResultLog) { + loggedMddLibApiResultLog.add(mddLibApiResultLog); + } + + public List<Void> getLoggedMddLibApiResultLogs() { + return loggedMddLibApiResultLog; + } + + @Override + public ListenableFuture<Void> logMddStorageStats( + AsyncCallable<MddStorageStats> buildMddStorageStats) { return immediateFailedFuture( new UnsupportedOperationException("This method is not implemented in the fake yet.")); } @@ -82,7 +101,7 @@ public final class FakeEventLogger implements EventLogger { @Override public void logMddNetworkSavings( - Void fileGroupDetails, + DataDownloadFileGroupStats fileGroupDetails, int code, long fullFileSize, long downloadedFileSize, @@ -92,13 +111,14 @@ public final class FakeEventLogger implements EventLogger { } @Override - public void logMddDownloadResult(int code, Void fileGroupDetails) { + public void logMddDownloadResult( + MddDownloadResult.Code code, DataDownloadFileGroupStats fileGroupDetails) { throw new UnsupportedOperationException("This method is not implemented in the fake yet."); } @Override - public void logMddQueryStats(Void fileGroupDetails) { - throw new UnsupportedOperationException("This method is not implemented in the fake yet."); + public void logMddQueryStats(DataDownloadFileGroupStats fileGroupDetails) { + loggedMddQueryStats.add(fileGroupDetails); } @Override @@ -107,20 +127,43 @@ public final class FakeEventLogger implements EventLogger { } @Override - public void logMddDownloadLatency(Void fileGroupStats, Void downloadLatency) { + public void logMddDownloadLatency( + DataDownloadFileGroupStats fileGroupStats, Void downloadLatency) { loggedLatencies.put(fileGroupStats, downloadLatency); } @Override - public void logMddUsageEvent(Void fileGroupDetails, Void usageEventLog) { + public void logMddUsageEvent(DataDownloadFileGroupStats fileGroupDetails, Void usageEventLog) { throw new UnsupportedOperationException("This method is not implemented in the fake yet."); } - public List<Integer> getLoggedCodes() { + @Override + public void logNewConfigReceived( + DataDownloadFileGroupStats fileGroupDetails, Void newConfigReceivedInfo) { + loggedNewConfigReceived.put(fileGroupDetails, newConfigReceivedInfo); + } + + public void reset() { + loggedCodes.clear(); + loggedLatencies.clear(); + loggedMddQueryStats.clear(); + loggedNewConfigReceived.clear(); + loggedMddLibApiResultLog.clear(); + } + + public ArrayListMultimap<DataDownloadFileGroupStats, Void> getLoggedNewConfigReceived() { + return loggedNewConfigReceived; + } + + public List<MddClientEvent.Code> getLoggedCodes() { return loggedCodes; } - public ArrayListMultimap<Void, Void> getLoggedLatencies() { + public ArrayListMultimap<DataDownloadFileGroupStats, Void> getLoggedLatencies() { return loggedLatencies; } + + public ArrayList<DataDownloadFileGroupStats> getLoggedMddQueryStats() { + return loggedMddQueryStats; + } } diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/proto/BUILD b/java/com/google/android/libraries/mobiledatadownload/internal/proto/BUILD index d6f0c9d..6be1b57 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/proto/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/internal/proto/BUILD @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/proto/metadata.proto b/java/com/google/android/libraries/mobiledatadownload/internal/proto/metadata.proto index cab8a0f..406133c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/proto/metadata.proto +++ b/java/com/google/android/libraries/mobiledatadownload/internal/proto/metadata.proto @@ -41,7 +41,7 @@ message ExtraHttpHeader { // The tag number of extra fields should start from 1000 to reserve room for // growing DataFileGroup. // -// Next id: 1000 +// Next id: 1001 message DataFileGroupInternal { // Extra information that is kept on disk. // @@ -199,6 +199,15 @@ message DataFileGroupInternal { reserved 28; + // If a group enables preserve_filenames_and_isolate_files + // this property will contain the directory root of the isolated + // structure. Specifically, the property will be a string created from the + // group name and a hash of other identifying properties (account, variantid, + // buildid). + // + // currently only used in aMDD. + optional string isolated_directory_root = 1000; + reserved 4, 5, 7, 8, 9, 15, 18, 22, 24; } @@ -507,8 +516,23 @@ message GroupKey { // Whether or not all files in a fileGroup have been downloaded. optional bool downloaded = 4; - // The variant id of the group. A null or empty value indicates that the group - // does not have an associated variant. + // The variant id of the group for identification purposes. + // + // This is used to ensure that groups with different variants can have + // different entries in MDD metadata, and therefore have different lifecycles. + // + // Note that clients can choose to opt-in to a SINGLE_VARIANT flow where + // different variants replace each other on-device (only single variant can + // exist on a device at a time). In this case, an empty variant_id is set here + // so groups with different variants share the same GroupKey and are subject + // to the same lifecycle, even though the DataFileGroup does have a non-empty + // variant_id. + // + // Because of the SINGLE_VARIANT flow and because groups may still be added + // with no variant_id associated, using this property to tell if the + // associated file group has a variant_id is unreliable. Instead, the + // variant_id set within a DataFileGroup should be used as the source of truth + // about the group (such as when logging). optional string variant_id = 6; reserved 3; @@ -651,11 +675,26 @@ message LoggingState { // This proto is used to store state for logging that is specific to a File // Group. This includes network usage logging and maybe download tiers (for // <internal>). +// +// NEXT TAG: 7 message FileGroupLoggingState { + // GroupKey associated with a file group -- this is used to populate the group + // name and host package name. optional GroupKey group_key = 1; + + // The build_id associated with the file group. optional int64 build_id = 2; + + // The variant_id associated with the file group. + optional string variant_id = 6; + + // The file group version number associated with the file group. optional int32 file_group_version_number = 3; + + // The number of bytes downloaded over a cellular (metered) network. optional int64 cellular_usage = 4; + + // The number of bytes downloaded over a wifi (unmetered) network. optional int64 wifi_usage = 5; } diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/util/BUILD b/java/com/google/android/libraries/mobiledatadownload/internal/util/BUILD index de5e844..1ba22c1 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/util/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/internal/util/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -36,6 +37,18 @@ android_library( ) android_library( + name = "DownloadFutureMap", + srcs = ["DownloadFutureMap.java"], + deps = [ + "//java/com/google/android/libraries/mobiledatadownload/foreground:NotificationUtil", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "@androidx_core_core", + "@com_google_guava_guava", + ], +) + +android_library( name = "AndroidSharingUtil", srcs = ["AndroidSharingUtil.java"], deps = [ @@ -45,6 +58,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file/openers:stream", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "//proto:log_enums_java_proto_lite", "@com_google_guava_guava", ], ) @@ -60,6 +74,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/internal:MddConstants", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//proto:transform_java_proto_lite", + "//third_party/java/android_libs/guava_jdk5:hash", "@com_google_code_findbugs_jsr305", "@com_google_guava_guava", ], @@ -83,6 +98,7 @@ android_library( srcs = ["FuturesUtil.java"], deps = [ "//java/com/google/android/libraries/mobiledatadownload/internal/annotations:SequentialControlExecutor", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "@com_google_guava_guava", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/util/DirectoryUtil.java b/java/com/google/android/libraries/mobiledatadownload/internal/util/DirectoryUtil.java index 822b421..8ccd20b 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/util/DirectoryUtil.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/util/DirectoryUtil.java @@ -108,6 +108,7 @@ public class DirectoryUtil { * URI, otherwise it returns the "android" scheme URI. */ // TODO(b/118137672): getOnDeviceUri shouldn't return null on error. + @Nullable public static Uri getOnDeviceUri( Context context, diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/util/DownloadFutureMap.java b/java/com/google/android/libraries/mobiledatadownload/internal/util/DownloadFutureMap.java new file mode 100644 index 0000000..81c354f --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/internal/util/DownloadFutureMap.java @@ -0,0 +1,127 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.internal.util; + +import static com.google.common.util.concurrent.Futures.immediateFailedFuture; +import static com.google.common.util.concurrent.Futures.immediateVoidFuture; + +import androidx.annotation.VisibleForTesting; +import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedExecutionSequencer; +import com.google.common.base.Optional; +import com.google.common.util.concurrent.ListenableFuture; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Executor; + +/** + * Helper class to maintain the state of MDD download futures. + * + * <p>This follows a limited Map interface and uses {@link ExecutionSequencer} to ensure that all + * operations on the map are synchronized. + * + * <p><b>NOTE:</b> This class is meant to be a container class for download futures and <em>should + * not</em> include any download-specific logic. Its sole purpose is to maintain any in-progress + * download futures in a synchronized manner. Download-specific logic should be implemented outside + * of this class, and can rely on {@link StateChangeCallbacks} to respond to events from this map. + */ +public final class DownloadFutureMap<T> { + private static final String TAG = "DownloadFutureMap"; + + // ExecutionSequencer ensures that enqueued futures are executed sequentially (regardless of the + // executor used). This allows us to keep critical state changes sequential. + private final PropagatedExecutionSequencer futureSerializer = + PropagatedExecutionSequencer.create(); + + private final Executor sequentialControlExecutor; + private final StateChangeCallbacks callbacks; + + // Underlying map to store futures -- synchronization of accesses/updates is handled by + // ExecutionSequencer. + @VisibleForTesting + public final Map<String, ListenableFuture<T>> keyToDownloadFutureMap = new HashMap<>(); + + private DownloadFutureMap(Executor sequentialControlExecutor, StateChangeCallbacks callbacks) { + this.sequentialControlExecutor = sequentialControlExecutor; + this.callbacks = callbacks; + } + + /** Convenience creator when no callbacks should be registered. */ + public static <T> DownloadFutureMap<T> create(Executor sequentialControlExecutor) { + return create(sequentialControlExecutor, new StateChangeCallbacks() {}); + } + + /** Creates a new instance of DownloadFutureMap. */ + public static <T> DownloadFutureMap<T> create( + Executor sequentialControlExecutor, StateChangeCallbacks callbacks) { + return new DownloadFutureMap<T>(sequentialControlExecutor, callbacks); + } + + /** Callback to support custom events based on the state of the map. */ + public static interface StateChangeCallbacks { + /** Respond to the event immediately before a new future is added to the map. */ + default void onAdd(String key, int newSize) throws Exception {} + + /** Respond to the event immediately after a future is removed from the map. */ + default void onRemove(String key, int newSize) throws Exception {} + } + + public ListenableFuture<Void> add(String key, ListenableFuture<T> downloadFuture) { + LogUtil.v("%s: submitting request to add in-progress download future with key: %s", TAG, key); + return futureSerializer.submitAsync( + () -> { + try { + callbacks.onAdd(key, keyToDownloadFutureMap.size() + 1); + keyToDownloadFutureMap.put(key, downloadFuture); + } catch (Exception e) { + LogUtil.e(e, "%s: Failed to add download future (%s) to map", TAG, key); + return immediateFailedFuture(e); + } + return immediateVoidFuture(); + }, + sequentialControlExecutor); + } + + @SuppressWarnings("FutureReturnValueIgnored") + public ListenableFuture<Void> remove(String key) { + LogUtil.v( + "%s: submitting request to remove in-progress download future with key: %s", TAG, key); + return futureSerializer.submitAsync( + () -> { + try { + keyToDownloadFutureMap.remove(key); + callbacks.onRemove(key, keyToDownloadFutureMap.size()); + } catch (Exception e) { + LogUtil.e(e, "%s: Failed to remove download future (%s) from map", TAG, key); + return immediateFailedFuture(e); + } + return immediateVoidFuture(); + }, + sequentialControlExecutor); + } + + public ListenableFuture<Optional<ListenableFuture<T>>> get(String key) { + LogUtil.v("%s: submitting request for in-progress download future with key: %s", TAG, key); + return futureSerializer.submit( + () -> Optional.fromNullable(keyToDownloadFutureMap.get(key)), sequentialControlExecutor); + } + + public ListenableFuture<Boolean> containsKey(String key) { + LogUtil.v("%s: submitting check for in-progress download future with key: %s", TAG, key); + return futureSerializer.submit( + () -> keyToDownloadFutureMap.containsKey(key), sequentialControlExecutor); + } +} diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/util/FileGroupUtil.java b/java/com/google/android/libraries/mobiledatadownload/internal/util/FileGroupUtil.java index eed5da0..63bf9a0 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/util/FileGroupUtil.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/util/FileGroupUtil.java @@ -28,6 +28,8 @@ import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.ImmutableSet; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; import com.google.mobiledatadownload.TransformProto.Transform; import com.google.mobiledatadownload.internal.MetadataProto.DataFile; import com.google.mobiledatadownload.internal.MetadataProto.DataFile.AndroidSharingType; @@ -51,9 +53,7 @@ public class FileGroupUtil { : TimeUnit.SECONDS.toMillis(fileGroup.getExpirationDateSecs()); } - /** - * @return the expiration date of this stale file group in millis - */ + /** Returns the expiration date of this stale file group in millis. */ public static long getStaleExpirationDateMillis(DataFileGroupInternal fileGroup) { return TimeUnit.SECONDS.toMillis(fileGroup.getBookkeeping().getStaleExpirationDate()); } @@ -151,6 +151,29 @@ public class FileGroupUtil { return dataFileGroup; } + /** Sets the isolated root if the file group supports isolated structures. */ + public static DataFileGroupInternal maybeSetIsolatedRoot( + DataFileGroupInternal dataFileGroup, GroupKey groupKey) { + // Check if isolated structure is allowed before adding the root + if (!isIsolatedStructureAllowed(dataFileGroup)) { + return dataFileGroup; + } + + Hasher isolatedRootHasher = + Hashing.sha256() + .newHasher() + .putUnencodedChars(dataFileGroup.getVariantId()) + .putUnencodedChars(MddConstants.SPLIT_CHAR) + .putUnencodedChars(groupKey.getAccount()) + .putUnencodedChars(MddConstants.SPLIT_CHAR) + .putLong(dataFileGroup.getBuildId()); + + String hash = isolatedRootHasher.hash().toString(); + String directoryRoot = String.format("%s_%s", dataFileGroup.getGroupName(), hash); + + return dataFileGroup.toBuilder().setIsolatedDirectoryRoot(directoryRoot).build(); + } + /** Shared method to test whether the given file group supports isolated file structures. */ public static boolean isIsolatedStructureAllowed(DataFileGroupInternal dataFileGroupInternal) { if (VERSION.SDK_INT < VERSION_CODES.LOLLIPOP @@ -174,10 +197,20 @@ public class FileGroupUtil { */ public static Uri getIsolatedRootDirectory( Context context, Optional<String> instanceId, DataFileGroupInternal fileGroupInternal) { + String groupRoot; + if (!fileGroupInternal.getIsolatedDirectoryRoot().isEmpty()) { + groupRoot = fileGroupInternal.getIsolatedDirectoryRoot(); + } else { + // NOTE: Only the group name was used before the isolated directory root field was + // added. To preserve backwards compatibility, fallback to group name if isolated directory + // root is not present. + groupRoot = fileGroupInternal.getGroupName(); + } + return DirectoryUtil.getDownloadSymlinkDirectory( context, fileGroupInternal.getAllowedReadersEnum(), instanceId) .buildUpon() - .appendPath(fileGroupInternal.getGroupName()) + .appendPath(groupRoot) .build(); } @@ -190,8 +223,13 @@ public class FileGroupUtil { Optional<String> instanceId, DataFile dataFile, DataFileGroupInternal parentFileGroup) { - Uri.Builder fileUriBuilder = - getIsolatedRootDirectory(context, instanceId, parentFileGroup).buildUpon(); + Uri rootUri = getIsolatedRootDirectory(context, instanceId, parentFileGroup); + return appendIsolatedFileUri(rootUri, dataFile); + } + + /** Helper method to append isolated file uri to an already known root. */ + public static Uri appendIsolatedFileUri(Uri rootUri, DataFile dataFile) { + Uri.Builder fileUriBuilder = rootUri.buildUpon(); if (dataFile.getRelativeFilePath().isEmpty()) { // If no relative path specified get the last segment from the // urlToDownload. @@ -223,7 +261,8 @@ public class FileGroupUtil { Uri isolatedRootDir = FileGroupUtil.getIsolatedRootDirectory(context, instanceId, dataFileGroup); if (fileStorage.exists(isolatedRootDir)) { - Void unused = fileStorage.open(isolatedRootDir, RecursiveDeleteOpener.create()); + Void unused = + fileStorage.open(isolatedRootDir, RecursiveDeleteOpener.create().withNoFollowLinks()); } } @@ -257,24 +296,29 @@ public class FileGroupUtil { public static boolean isSideloadedFile(DataFile dataFile) { return isFileWithMatchingScheme( - dataFile, + dataFile.getUrlToDownload(), ImmutableSet.of( MddConstants.SIDELOAD_FILE_URL_SCHEME, MddConstants.EMBEDDED_ASSET_URL_SCHEME)); } public static boolean isInlineFile(DataFile dataFile) { - return isFileWithMatchingScheme(dataFile, ImmutableSet.of(MddConstants.INLINE_FILE_URL_SCHEME)); + return isFileWithMatchingScheme( + dataFile.getUrlToDownload(), ImmutableSet.of(MddConstants.INLINE_FILE_URL_SCHEME)); + } + + public static boolean isInlineFile(String url) { + return isFileWithMatchingScheme(url, ImmutableSet.of(MddConstants.INLINE_FILE_URL_SCHEME)); } // Helper method to test whether a DataFile's url scheme is contained in the given scheme set. - private static boolean isFileWithMatchingScheme(DataFile dataFile, ImmutableSet<String> schemes) { - if (!dataFile.hasUrlToDownload()) { + private static boolean isFileWithMatchingScheme(String url, ImmutableSet<String> schemes) { + if (url.isEmpty()) { return false; } - int colon = dataFile.getUrlToDownload().indexOf(':'); + int colon = url.indexOf(':'); // TODO(b/196593240): Ensure this is always handled, or replace with a checked exception - Preconditions.checkState(colon > -1, "Invalid url: %s", dataFile.getUrlToDownload()); - String fileScheme = dataFile.getUrlToDownload().substring(0, colon); + Preconditions.checkState(colon > -1, "Invalid url: %s", url); + String fileScheme = url.substring(0, colon); for (String scheme : schemes) { if (Ascii.equalsIgnoreCase(fileScheme, scheme)) { return true; diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/util/FileGroupsMetadataUtil.java b/java/com/google/android/libraries/mobiledatadownload/internal/util/FileGroupsMetadataUtil.java index fda3b1e..2948df6 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/util/FileGroupsMetadataUtil.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/util/FileGroupsMetadataUtil.java @@ -20,9 +20,9 @@ import android.util.Base64; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; +import com.google.protobuf.InvalidProtocolBufferException; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; -import com.google.protobuf.InvalidProtocolBufferException; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -94,7 +94,7 @@ public final class FileGroupsMetadataUtil { } // TODO(b/129702287): Move away from proto based serialization. - public static String getSerializedGroupKey(GroupKey groupKey, Context context) { + public static String getSerializedGroupKey(GroupKey groupKey) { byte[] byteValue = groupKey.toByteArray(); return Base64.encodeToString(byteValue, Base64.NO_PADDING | Base64.NO_WRAP); } @@ -102,7 +102,8 @@ public final class FileGroupsMetadataUtil { /** * Converts a string representing a serialized GroupKey into a GroupKey. * - * @return - groupKey if able to parse stringKey properly. null if parsing fails. + * @return groupKey if able to parse string key properly. + * @throws GroupKeyDeserializationException when unable to parse string key */ // TODO(b/129702287): Move away from proto based deserialization. public static GroupKey deserializeGroupKey(String serializedGroupKey) @@ -110,7 +111,7 @@ public final class FileGroupsMetadataUtil { try { return SharedPreferencesUtil.parseLiteFromEncodedString( serializedGroupKey, GroupKey.parser()); - } catch (InvalidProtocolBufferException e) { + } catch (NullPointerException | InvalidProtocolBufferException e) { throw new GroupKeyDeserializationException( "Failed to deserialize key:" + serializedGroupKey, e); } diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/util/FuturesUtil.java b/java/com/google/android/libraries/mobiledatadownload/internal/util/FuturesUtil.java index cdf1ea3..0e0013c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/util/FuturesUtil.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/util/FuturesUtil.java @@ -15,10 +15,13 @@ */ package com.google.android.libraries.mobiledatadownload.internal.util; +import static com.google.common.util.concurrent.Futures.immediateFuture; + import com.google.android.libraries.mobiledatadownload.internal.annotations.SequentialControlExecutor; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Function; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.util.ArrayList; import java.util.List; import java.util.concurrent.Executor; @@ -89,18 +92,20 @@ public final class FuturesUtil { this.init = init; } + @CanIgnoreReturnValue public SequentialFutureChain<T> chain(Function<T, T> operation) { operations.add(new DirectFutureChainElement<>(operation)); return this; } + @CanIgnoreReturnValue public SequentialFutureChain<T> chainAsync(Function<T, ListenableFuture<T>> operation) { operations.add(new AsyncFutureChainElement<>(operation)); return this; } public ListenableFuture<T> start() { - ListenableFuture<T> result = Futures.immediateFuture(init); + ListenableFuture<T> result = immediateFuture(init); for (FutureChainElement<T> operation : operations) { result = operation.apply(result); } @@ -121,7 +126,7 @@ public final class FuturesUtil { @Override public ListenableFuture<T> apply(ListenableFuture<T> input) { - return Futures.transform(input, operation::apply, sequentialExecutor); + return PropagatedFutures.transform(input, operation, sequentialExecutor); } } @@ -134,7 +139,7 @@ public final class FuturesUtil { @Override public ListenableFuture<T> apply(ListenableFuture<T> input) { - return Futures.transformAsync(input, operation::apply, sequentialExecutor); + return PropagatedFutures.transformAsync(input, operation::apply, sequentialExecutor); } } } diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/util/ProtoConversionUtil.java b/java/com/google/android/libraries/mobiledatadownload/internal/util/ProtoConversionUtil.java index 04e3446..cdc8a58 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/util/ProtoConversionUtil.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/util/ProtoConversionUtil.java @@ -41,6 +41,13 @@ public final class ProtoConversionUtil { group.toByteArray(), ExtensionRegistryLite.getEmptyRegistry()); } + public static DataFileGroup reverse(DataFileGroupInternal group) + throws InvalidProtocolBufferException { + // Cannot use generated registry here, because it may cause NPE to clients. + // For more detail, see b/140135059. + return DataFileGroup.parseFrom(group.toByteArray(), ExtensionRegistryLite.getEmptyRegistry()); + } + /** * Converts external proto {@link DownloadConditions} into internal proto {@link * MetadataProto.DownloadConditions}. @@ -61,6 +68,10 @@ public final class ProtoConversionUtil { // TODO(b/176103639): Use automated proto converter instead // LINT.IfChange(data_file_convert) public static MetadataProto.DataFile convertDataFile(DataFile dataFile) { + // incompatible argument for parameter value of setChecksumType. + // incompatible argument for parameter value of setAndroidSharingType. + // incompatible argument for parameter value of setAndroidSharingChecksumType. + @SuppressWarnings("nullness:argument.type.incompatible") MetadataProto.DataFile.Builder dataFileBuilder = MetadataProto.DataFile.newBuilder() .setFileId(dataFile.getFileId()) @@ -110,6 +121,8 @@ public final class ProtoConversionUtil { */ // TODO(b/176103639): Use automated proto converter instead // LINT.IfChange(delta_file_convert) + // incompatible argument for parameter value of setDiffDecoder. + @SuppressWarnings("nullness:argument.type.incompatible") public static MetadataProto.DeltaFile convertDeltaFile(DeltaFile deltaFile) { return MetadataProto.DeltaFile.newBuilder() .setUrlToDownload(deltaFile.getUrlToDownload()) diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/util/SharedFilesMetadataUtil.java b/java/com/google/android/libraries/mobiledatadownload/internal/util/SharedFilesMetadataUtil.java index 323819b..7f7cff6 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/util/SharedFilesMetadataUtil.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/util/SharedFilesMetadataUtil.java @@ -93,6 +93,8 @@ public final class SharedFilesMetadataUtil { .toString(); } + // incompatible argument for parameter value of setAllowedReaders. + @SuppressWarnings("nullness:argument.type.incompatible") public static NewFileKey deserializeNewFileKey( String serializedFileKey, Context context, SilentFeedback silentFeedback) throws FileKeyDeserializationException { diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/util/SymlinkUtil.java b/java/com/google/android/libraries/mobiledatadownload/internal/util/SymlinkUtil.java index 9ec91e8..ba4dc3d 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/util/SymlinkUtil.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/util/SymlinkUtil.java @@ -29,6 +29,7 @@ import com.google.android.libraries.mobiledatadownload.file.common.MalformedUriE import java.io.IOException; /** Utility class to create symlinks (if supported). */ +@RequiresApi(VERSION_CODES.LOLLIPOP) public final class SymlinkUtil { private SymlinkUtil() {} diff --git a/java/com/google/android/libraries/mobiledatadownload/lite/BUILD b/java/com/google/android/libraries/mobiledatadownload/lite/BUILD index c1eb8fb..4f8c1f5 100644 --- a/java/com/google/android/libraries/mobiledatadownload/lite/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/lite/BUILD @@ -16,6 +16,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") # MDD Lite visibility is restricted to the following set of packages. Any # new clients must be added to this list in order to grant build visibility. package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -37,11 +38,15 @@ android_library( ":DownloadProgressMonitor", "//java/com/google/android/libraries/mobiledatadownload:DownloadException", "//java/com/google/android/libraries/mobiledatadownload/downloader:FileDownloader", + "//java/com/google/android/libraries/mobiledatadownload/foreground:ForegroundDownloadKey", "//java/com/google/android/libraries/mobiledatadownload/foreground:NotificationUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", + "//java/com/google/android/libraries/mobiledatadownload/internal/util:DownloadFutureMap", + "//java/com/google/android/libraries/mobiledatadownload/tracing", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "@androidx_core_core", "@com_google_auto_value", - "@com_google_dagger", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", "@org_checkerframework_qual", ], @@ -66,9 +71,11 @@ android_library( ":DownloadListener", "//java/com/google/android/libraries/mobiledatadownload:TimeSource", "//java/com/google/android/libraries/mobiledatadownload/file/spi", + "//java/com/google/android/libraries/mobiledatadownload/internal:AndroidTimeSource", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", "@androidx_annotation_annotation", "@com_google_code_findbugs_jsr305", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/lite/DownloadProgressMonitor.java b/java/com/google/android/libraries/mobiledatadownload/lite/DownloadProgressMonitor.java index d0fd6fa..4c5fbd6 100644 --- a/java/com/google/android/libraries/mobiledatadownload/lite/DownloadProgressMonitor.java +++ b/java/com/google/android/libraries/mobiledatadownload/lite/DownloadProgressMonitor.java @@ -21,11 +21,11 @@ import com.google.android.libraries.mobiledatadownload.TimeSource; import com.google.android.libraries.mobiledatadownload.file.spi.Monitor; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.common.util.concurrent.MoreExecutors; +import com.google.errorprone.annotations.concurrent.GuardedBy; import java.util.HashMap; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicLong; import javax.annotation.Nullable; -import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.ThreadSafe; /** A Download Progress Monitor to support {@link DownloadListener}. */ @@ -37,7 +37,6 @@ public class DownloadProgressMonitor implements Monitor, SingleFileDownloadProgr private final TimeSource timeSource; private final Executor sequentialControlExecutor; - // NOTE: GuardRails prohibits multiple public constructors private DownloadProgressMonitor(TimeSource timeSource, Executor controlExecutor) { this.timeSource = timeSource; diff --git a/java/com/google/android/libraries/mobiledatadownload/lite/Downloader.java b/java/com/google/android/libraries/mobiledatadownload/lite/Downloader.java index 208132c..ea4f450 100644 --- a/java/com/google/android/libraries/mobiledatadownload/lite/Downloader.java +++ b/java/com/google/android/libraries/mobiledatadownload/lite/Downloader.java @@ -22,6 +22,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Supplier; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.errorprone.annotations.CheckReturnValue; import java.util.concurrent.Executor; @@ -73,8 +74,19 @@ public interface Downloader { @CheckReturnValue ListenableFuture<Void> downloadWithForegroundService(DownloadRequest downloadRequest); - /** Cancel an on-going foreground download. */ - void cancelForegroundDownload(String destinationFileUri); + /** + * Cancel an on-going foreground download. + * + * <p>Use {@link ForegroundDownloadKey} to construct the unique key. + * + * <p><b>NOTE:</b> In most cases, clients will not need to call this -- it is meant to allow the + * ForegroundDownloadService to cancel a download via the Cancel action registered to a + * notification. + * + * <p>Clients should prefer to cancel the future returned to them from {@link + * #downloadWithForegroundService} instead. + */ + void cancelForegroundDownload(String downloadKey); static Downloader.Builder newBuilder() { return new Downloader.Builder(); @@ -91,12 +103,14 @@ public interface Downloader { private Optional<SingleFileDownloadProgressMonitor> downloadMonitorOptional = Optional.absent(); private Optional<Class<?>> foregroundDownloadServiceClassOptional = Optional.absent(); + @CanIgnoreReturnValue public Builder setContext(Context context) { this.context = context.getApplicationContext(); return this; } /** Set the Control Executor which will run MDDLite control flow. */ + @CanIgnoreReturnValue public Builder setControlExecutor(Executor controlExecutor) { Preconditions.checkNotNull(controlExecutor); // Executor that will execute tasks sequentially. @@ -115,6 +129,7 @@ public interface Downloader { * DownloadListener} to {@link Downloader#download}. The DownloadListener's {@code onFailure} * and {@code onComplete} will be invoked regardless of whether this is set. */ + @CanIgnoreReturnValue public Builder setDownloadMonitor(SingleFileDownloadProgressMonitor downloadMonitor) { this.downloadMonitorOptional = Optional.of(downloadMonitor); return this; @@ -127,6 +142,7 @@ public interface Downloader { * <p>This is required to use {@link Downloader#downloadWithForegroundService}. Not providing * this will result in a failed future when calling downloadWithForegroundService. */ + @CanIgnoreReturnValue public Builder setForegroundDownloadService(Class<?> foregroundDownloadServiceClass) { this.foregroundDownloadServiceClassOptional = Optional.of(foregroundDownloadServiceClass); return this; @@ -136,6 +152,7 @@ public interface Downloader { * Set the FileDownloader Supplier. MDDLite takes in a Supplier of FileDownload to support lazy * instantiation of the FileDownloader */ + @CanIgnoreReturnValue public Builder setFileDownloaderSupplier(Supplier<FileDownloader> fileDownloaderSupplier) { this.fileDownloaderSupplier = fileDownloaderSupplier; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/lite/DownloaderImpl.java b/java/com/google/android/libraries/mobiledatadownload/lite/DownloaderImpl.java index 1c0cb49..8472667 100644 --- a/java/com/google/android/libraries/mobiledatadownload/lite/DownloaderImpl.java +++ b/java/com/google/android/libraries/mobiledatadownload/lite/DownloaderImpl.java @@ -15,6 +15,9 @@ */ package com.google.android.libraries.mobiledatadownload.lite; +import static com.google.common.util.concurrent.Futures.immediateFailedFuture; +import static com.google.common.util.concurrent.Futures.immediateVoidFuture; + import android.content.Context; import androidx.annotation.VisibleForTesting; import androidx.core.app.NotificationCompat; @@ -22,17 +25,18 @@ import androidx.core.app.NotificationManagerCompat; import com.google.android.libraries.mobiledatadownload.DownloadException; import com.google.android.libraries.mobiledatadownload.DownloadException.DownloadResultCode; import com.google.android.libraries.mobiledatadownload.downloader.FileDownloader; +import com.google.android.libraries.mobiledatadownload.foreground.ForegroundDownloadKey; import com.google.android.libraries.mobiledatadownload.foreground.NotificationUtil; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; +import com.google.android.libraries.mobiledatadownload.internal.util.DownloadFutureMap; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentFuture; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Optional; -import com.google.common.base.Preconditions; import com.google.common.base.Supplier; import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListenableFutureTask; import com.google.common.util.concurrent.MoreExecutors; -import java.util.HashMap; -import java.util.Map; import java.util.concurrent.Executor; import org.checkerframework.checker.nullness.compatqual.NullableDecl; @@ -46,9 +50,8 @@ final class DownloaderImpl implements Downloader { private final Optional<SingleFileDownloadProgressMonitor> downloadMonitorOptional; private final Supplier<FileDownloader> fileDownloaderSupplier; - // Synchronization will be done through sequentialControlExecutor - @VisibleForTesting - final Map<String, ListenableFuture<Void>> keyToListenableFuture = new HashMap<>(); + @VisibleForTesting final DownloadFutureMap<Void> downloadFutureMap; + @VisibleForTesting final DownloadFutureMap<Void> foregroundDownloadFutureMap; DownloaderImpl( Context context, @@ -61,19 +64,25 @@ final class DownloaderImpl implements Downloader { this.foregroundDownloadServiceClassOptional = foregroundDownloadServiceClassOptional; this.downloadMonitorOptional = downloadMonitorOptional; this.fileDownloaderSupplier = fileDownloaderSupplier; + this.downloadFutureMap = DownloadFutureMap.create(sequentialControlExecutor); + this.foregroundDownloadFutureMap = + DownloadFutureMap.create( + sequentialControlExecutor, + createCallbacksForForegroundService(context, foregroundDownloadServiceClassOptional)); } @Override public ListenableFuture<Void> download(DownloadRequest downloadRequest) { LogUtil.d("%s: download for Uri = %s", TAG, downloadRequest.destinationFileUri().toString()); - return Futures.submitAsync( - () -> { + ForegroundDownloadKey foregroundDownloadKey = + ForegroundDownloadKey.ofSingleFile(downloadRequest.destinationFileUri()); + + return PropagatedFutures.transformAsync( + getInProgressDownloadFuture(foregroundDownloadKey.toString()), + (Optional<ListenableFuture<Void>> existingDownloadFuture) -> { // if there is the same on-going request, return that one. - if (keyToListenableFuture.containsKey(downloadRequest.destinationFileUri().toString())) { - // uriToListenableFuture.get must return Non-null since we check the containsKey above. - // checkNotNull is to suppress false alarm about @Nullable result. - return Preconditions.checkNotNull( - keyToListenableFuture.get(downloadRequest.destinationFileUri().toString())); + if (existingDownloadFuture.isPresent()) { + return existingDownloadFuture.get(); } // Register listener with monitor if present @@ -87,13 +96,19 @@ final class DownloaderImpl implements Downloader { } else { LogUtil.w( "%s: download request included DownloadListener, but DownloadMonitor is not" - + " present! DownloadListener will only be invoked for complete/failure."); + + " present! DownloadListener will only be invoked for complete/failure.", + TAG); } } - ListenableFuture<Void> downloadFuture = startDownload(downloadRequest); + // Create a ListenableFutureTask to delay starting the downloadFuture until we can add the + // future to our map. + ListenableFutureTask<Void> startTask = ListenableFutureTask.create(() -> null); + ListenableFuture<Void> downloadFuture = + PropagatedFutures.transformAsync( + startTask, unused -> startDownload(downloadRequest), sequentialControlExecutor); - Futures.addCallback( + PropagatedFutures.addCallback( downloadFuture, new FutureCallback<Void>() { @Override @@ -104,36 +119,36 @@ final class DownloaderImpl implements Downloader { // Remove download listener and remove download future from map after listener // completes if (downloadRequest.listenerOptional().isPresent()) { - Futures.addCallback( + PropagatedFutures.addCallback( downloadRequest.listenerOptional().get().onComplete(), new FutureCallback<Void>() { @Override public void onSuccess(@NullableDecl Void result) { - keyToListenableFuture.remove( - downloadRequest.destinationFileUri().toString()); if (downloadMonitorOptional.isPresent()) { downloadMonitorOptional .get() .removeDownloadListener(downloadRequest.destinationFileUri()); } + ListenableFuture<Void> unused = + downloadFutureMap.remove(foregroundDownloadKey.toString()); } @Override public void onFailure(Throwable t) { LogUtil.e(t, "%s: Failed to run client onComplete", TAG); - keyToListenableFuture.remove( - downloadRequest.destinationFileUri().toString()); if (downloadMonitorOptional.isPresent()) { downloadMonitorOptional .get() .removeDownloadListener(downloadRequest.destinationFileUri()); } + ListenableFuture<Void> unused = + downloadFutureMap.remove(foregroundDownloadKey.toString()); } }, sequentialControlExecutor); } else { - // remove from future map immediately - keyToListenableFuture.remove(downloadRequest.destinationFileUri().toString()); + ListenableFuture<Void> unused = + downloadFutureMap.remove(foregroundDownloadKey.toString()); } } @@ -151,14 +166,20 @@ final class DownloaderImpl implements Downloader { .removeDownloadListener(downloadRequest.destinationFileUri()); } } - keyToListenableFuture.remove(downloadRequest.destinationFileUri().toString()); + ListenableFuture<Void> unused = + downloadFutureMap.remove(foregroundDownloadKey.toString()); } }, MoreExecutors.directExecutor()); - keyToListenableFuture.put( - downloadRequest.destinationFileUri().toString(), downloadFuture); - return downloadFuture; + return PropagatedFutures.transformAsync( + downloadFutureMap.add(foregroundDownloadKey.toString(), downloadFuture), + unused -> { + // Now that the download future is added, start the task and return the future + startTask.run(); + return downloadFuture; + }, + sequentialControlExecutor); }, sequentialControlExecutor); } @@ -178,7 +199,7 @@ final class DownloaderImpl implements Downloader { return fileDownloaderSupplier.get().startDownloading(fileDownloaderRequest); } catch (RuntimeException e) { // Catch any unchecked exceptions that prevented the download from starting. - return Futures.immediateFailedFuture( + return immediateFailedFuture( DownloadException.builder() .setDownloadResultCode(DownloadResultCode.UNKNOWN_ERROR) .setCause(e) @@ -192,23 +213,25 @@ final class DownloaderImpl implements Downloader { "%s: downloadWithForegroundService for Uri = %s", TAG, downloadRequest.destinationFileUri().toString()); if (!downloadMonitorOptional.isPresent()) { - return Futures.immediateFailedFuture( + return immediateFailedFuture( new IllegalStateException( "downloadWithForegroundService: DownloadMonitor is not provided!")); } if (!foregroundDownloadServiceClassOptional.isPresent()) { - return Futures.immediateFailedFuture( + return immediateFailedFuture( new IllegalStateException( "downloadWithForegroundService: ForegroundDownloadService is not provided!")); } - return Futures.submitAsync( - () -> { + + ForegroundDownloadKey foregroundDownloadKey = + ForegroundDownloadKey.ofSingleFile(downloadRequest.destinationFileUri()); + + return PropagatedFutures.transformAsync( + getInProgressDownloadFuture(foregroundDownloadKey.toString()), + (Optional<ListenableFuture<Void>> existingDownloadFuture) -> { // if there is the same on-going request, return that one. - if (keyToListenableFuture.containsKey(downloadRequest.destinationFileUri().toString())) { - // uriToListenableFuture.get must return Non-null since we check the containsKey above. - // checkNotNull is to suppress false alarm about @Nullable result. - return Preconditions.checkNotNull( - keyToListenableFuture.get(downloadRequest.destinationFileUri().toString())); + if (existingDownloadFuture.isPresent()) { + return existingDownloadFuture.get(); } // It's OK to recreate the NotificationChannel since it can also be used to restore a @@ -216,14 +239,6 @@ final class DownloaderImpl implements Downloader { // importance. NotificationUtil.createNotificationChannel(context); - // Only start the foreground download service when there is the first download request. - if (keyToListenableFuture.isEmpty()) { - NotificationUtil.startForegroundDownloadService( - context, - foregroundDownloadServiceClassOptional.get(), - downloadRequest.destinationFileUri().toString()); - } - DownloadListener downloadListenerWithNotification = createDownloadListenerWithNotification(downloadRequest); @@ -233,9 +248,14 @@ final class DownloaderImpl implements Downloader { .addDownloadListener( downloadRequest.destinationFileUri(), downloadListenerWithNotification); - ListenableFuture<Void> downloadFuture = startDownload(downloadRequest); + // Create a ListenableFutureTask to delay starting the downloadFuture until we can add the + // future to our map. + ListenableFutureTask<Void> startTask = ListenableFutureTask.create(() -> null); + ListenableFuture<Void> downloadFuture = + PropagatedFutures.transformAsync( + startTask, unused -> startDownload(downloadRequest), sequentialControlExecutor); - Futures.addCallback( + PropagatedFutures.addCallback( downloadFuture, new FutureCallback<Void>() { @Override @@ -243,7 +263,7 @@ final class DownloaderImpl implements Downloader { // Currently the MobStore monitor does not support onSuccess so we have to add // callback to the download future here. - Futures.addCallback( + PropagatedFutures.addCallback( downloadListenerWithNotification.onComplete(), new FutureCallback<Void>() { @Override @@ -267,15 +287,25 @@ final class DownloaderImpl implements Downloader { }, MoreExecutors.directExecutor()); - keyToListenableFuture.put( - downloadRequest.destinationFileUri().toString(), downloadFuture); - return downloadFuture; + return PropagatedFutures.transformAsync( + foregroundDownloadFutureMap.add(foregroundDownloadKey.toString(), downloadFuture), + unused -> { + // Now that the download future is added, start the task and return the future + startTask.run(); + return downloadFuture; + }, + sequentialControlExecutor); }, sequentialControlExecutor); } // Assertion: foregroundDownloadService and downloadMonitor are present private DownloadListener createDownloadListenerWithNotification(DownloadRequest downloadRequest) { + String networkPausedMessage = + downloadRequest.downloadConstraints().requireUnmeteredNetwork() + ? NotificationUtil.getDownloadPausedWifiMessage(context) + : NotificationUtil.getDownloadPausedMessage(context); + NotificationManagerCompat notificationManager = NotificationManagerCompat.from(context); NotificationCompat.Builder notification = NotificationUtil.createNotificationBuilder( @@ -284,14 +314,16 @@ final class DownloaderImpl implements Downloader { downloadRequest.notificationContentTitle(), downloadRequest.notificationContentTextOptional().or(downloadRequest.urlToDownload())); - int notificationKey = - NotificationUtil.notificationKeyForKey(downloadRequest.destinationFileUri().toString()); + ForegroundDownloadKey foregroundDownloadKey = + ForegroundDownloadKey.ofSingleFile(downloadRequest.destinationFileUri()); + + int notificationKey = NotificationUtil.notificationKeyForKey(foregroundDownloadKey.toString()); // Attach the Cancel action to the notification. NotificationUtil.createCancelAction( context, foregroundDownloadServiceClassOptional.get(), - downloadRequest.destinationFileUri().toString(), + foregroundDownloadKey.toString(), notification, notificationKey); notificationManager.notify(notificationKey, notification.build()); @@ -299,49 +331,56 @@ final class DownloaderImpl implements Downloader { return new DownloadListener() { @Override public void onProgress(long currentSize) { - sequentialControlExecutor.execute( - () -> { - // There can be a race condition, where onPausedForConnectivity can be called - // after onComplete or onFailure which removes the future and the notification. - if (keyToListenableFuture.containsKey( - downloadRequest.destinationFileUri().toString())) { - notification - .setCategory(NotificationCompat.CATEGORY_PROGRESS) - .setSmallIcon(android.R.drawable.stat_sys_download) - .setProgress( - downloadRequest.fileSizeBytes(), - (int) currentSize, - /* indeterminate = */ downloadRequest.fileSizeBytes() <= 0); - notificationManager.notify(notificationKey, notification.build()); - } - if (downloadRequest.listenerOptional().isPresent()) { - downloadRequest.listenerOptional().get().onProgress(currentSize); - } - }); + // TODO(b/229123693): return this future once DownloadListener has an async api. + ListenableFuture<?> unused = + PropagatedFutures.transformAsync( + foregroundDownloadFutureMap.containsKey(foregroundDownloadKey.toString()), + futureInProgress -> { + if (futureInProgress) { + notification + .setCategory(NotificationCompat.CATEGORY_PROGRESS) + .setContentText( + downloadRequest + .notificationContentTextOptional() + .or(downloadRequest.urlToDownload())) + .setSmallIcon(android.R.drawable.stat_sys_download) + .setProgress( + downloadRequest.fileSizeBytes(), + (int) currentSize, + /* indeterminate= */ downloadRequest.fileSizeBytes() <= 0); + notificationManager.notify(notificationKey, notification.build()); + } + if (downloadRequest.listenerOptional().isPresent()) { + downloadRequest.listenerOptional().get().onProgress(currentSize); + } + return immediateVoidFuture(); + }, + sequentialControlExecutor); } @Override public void onPausedForConnectivity() { - sequentialControlExecutor.execute( - () -> { - // There can be a race condition, where onPausedForConnectivity can be called - // after onComplete or onFailure which removes the future and the notification. - if (keyToListenableFuture.containsKey( - downloadRequest.destinationFileUri().toString())) { - notification - .setCategory(NotificationCompat.CATEGORY_STATUS) - .setContentText(NotificationUtil.getDownloadPausedMessage(context)) - .setSmallIcon(android.R.drawable.stat_sys_download) - .setOngoing(true) - // hide progress bar. - .setProgress(0, 0, false); - notificationManager.notify(notificationKey, notification.build()); - } - - if (downloadRequest.listenerOptional().isPresent()) { - downloadRequest.listenerOptional().get().onPausedForConnectivity(); - } - }); + // TODO(b/229123693): return this future once DownloadListener has an async api. + ListenableFuture<?> unused = + PropagatedFutures.transformAsync( + foregroundDownloadFutureMap.containsKey(foregroundDownloadKey.toString()), + futureInProgress -> { + if (futureInProgress) { + notification + .setCategory(NotificationCompat.CATEGORY_STATUS) + .setContentText(networkPausedMessage) + .setSmallIcon(android.R.drawable.stat_sys_download) + .setOngoing(true) + // hide progress bar. + .setProgress(0, 0, false); + notificationManager.notify(notificationKey, notification.build()); + } + if (downloadRequest.listenerOptional().isPresent()) { + downloadRequest.listenerOptional().get().onPausedForConnectivity(); + } + return immediateVoidFuture(); + }, + sequentialControlExecutor); } @Override @@ -350,92 +389,154 @@ final class DownloaderImpl implements Downloader { ListenableFuture<Void> clientOnCompleteFuture = downloadRequest.listenerOptional().isPresent() ? downloadRequest.listenerOptional().get().onComplete() - : Futures.immediateVoidFuture(); + : immediateVoidFuture(); // Logic to shutdown Foreground Download Service after the client's provided onComplete // finished - clientOnCompleteFuture.addListener( - () -> { - // Clear the notification action. - notification.mActions.clear(); - - if (downloadRequest.showDownloadedNotification()) { - notification - .setCategory(NotificationCompat.CATEGORY_STATUS) - .setContentText(NotificationUtil.getDownloadSuccessMessage(context)) - .setOngoing(false) - .setSmallIcon(android.R.drawable.stat_sys_download_done) - // hide progress bar. - .setProgress(0, 0, false); - - notificationManager.notify(notificationKey, notification.build()); - } else { - NotificationUtil.cancelNotificationForKey( - context, downloadRequest.destinationFileUri().toString()); - } - - keyToListenableFuture.remove(downloadRequest.destinationFileUri().toString()); - // If there is no other on-going foreground download, shutdown the - // ForegroundDownloadService - if (keyToListenableFuture.isEmpty()) { - NotificationUtil.stopForegroundDownloadService( - context, foregroundDownloadServiceClassOptional.get()); - } + return PropagatedFluentFuture.from(clientOnCompleteFuture) + .transformAsync( + unused -> { + // onComplete succeeded, show a success message + notification.mActions.clear(); + + if (downloadRequest.showDownloadedNotification()) { + notification + .setCategory(NotificationCompat.CATEGORY_STATUS) + .setContentText(NotificationUtil.getDownloadSuccessMessage(context)) + .setOngoing(false) + .setSmallIcon(android.R.drawable.stat_sys_download_done) + // hide progress bar. + .setProgress(0, 0, false); + + notificationManager.notify(notificationKey, notification.build()); + } else { + NotificationUtil.cancelNotificationForKey( + context, foregroundDownloadKey.toString()); + } + return immediateVoidFuture(); + }, + sequentialControlExecutor) + .catchingAsync( + Exception.class, + e -> { + LogUtil.w( + e, + "%s: Delegate onComplete failed for uri: %s, showing failure notification.", + TAG, + downloadRequest.destinationFileUri()); + notification.mActions.clear(); + + if (downloadRequest.showDownloadedNotification()) { + notification + .setCategory(NotificationCompat.CATEGORY_STATUS) + .setContentText(NotificationUtil.getDownloadFailedMessage(context)) + .setOngoing(false) + .setSmallIcon(android.R.drawable.stat_sys_warning) + // hide progress bar. + .setProgress(0, 0, false); + + notificationManager.notify(notificationKey, notification.build()); + } else { + NotificationUtil.cancelNotificationForKey( + context, downloadRequest.destinationFileUri().toString()); + } - downloadMonitorOptional - .get() - .removeDownloadListener(downloadRequest.destinationFileUri()); - }, - sequentialControlExecutor); - return clientOnCompleteFuture; + return immediateVoidFuture(); + }, + sequentialControlExecutor) + .transformAsync( + unused -> { + // After success or failure notification is shown, clean up + downloadMonitorOptional + .get() + .removeDownloadListener(downloadRequest.destinationFileUri()); + + return foregroundDownloadFutureMap.remove(foregroundDownloadKey.toString()); + }, + sequentialControlExecutor); } @Override public void onFailure(Throwable t) { - sequentialControlExecutor.execute( - () -> { - // Clear the notification action. - notification.mActions.clear(); - - // Show download failed in notification. - notification - .setCategory(NotificationCompat.CATEGORY_STATUS) - .setContentText(NotificationUtil.getDownloadFailedMessage(context)) - .setOngoing(false) - .setSmallIcon(android.R.drawable.stat_sys_warning) - // hide progress bar. - .setProgress(0, 0, false); - - notificationManager.notify(notificationKey, notification.build()); - - keyToListenableFuture.remove(downloadRequest.destinationFileUri().toString()); - - // If there is no other on-going foreground download, shutdown the - // ForegroundDownloadService - if (keyToListenableFuture.isEmpty()) { - NotificationUtil.stopForegroundDownloadService( - context, foregroundDownloadServiceClassOptional.get()); - } + // TODO(b/229123693): return this future once DownloadListener has an async api. + ListenableFuture<?> unused = + PropagatedFutures.submitAsync( + () -> { + // Clear the notification action. + notification.mActions.clear(); + + // Show download failed in notification. + notification + .setCategory(NotificationCompat.CATEGORY_STATUS) + .setContentText(NotificationUtil.getDownloadFailedMessage(context)) + .setOngoing(false) + .setSmallIcon(android.R.drawable.stat_sys_warning) + // hide progress bar. + .setProgress(0, 0, false); + + notificationManager.notify(notificationKey, notification.build()); - if (downloadRequest.listenerOptional().isPresent()) { - downloadRequest.listenerOptional().get().onFailure(t); - } - downloadMonitorOptional - .get() - .removeDownloadListener(downloadRequest.destinationFileUri()); - }); + if (downloadRequest.listenerOptional().isPresent()) { + downloadRequest.listenerOptional().get().onFailure(t); + } + downloadMonitorOptional + .get() + .removeDownloadListener(downloadRequest.destinationFileUri()); + + return foregroundDownloadFutureMap.remove(foregroundDownloadKey.toString()); + }, + sequentialControlExecutor); } }; } @Override - public void cancelForegroundDownload(String destinationFileUri) { - LogUtil.d("%s: CancelForegroundDownload for Uri = %s", TAG, destinationFileUri); - sequentialControlExecutor.execute( - () -> { - if (keyToListenableFuture.containsKey(destinationFileUri)) { - keyToListenableFuture.get(destinationFileUri).cancel(true); - } - }); + public void cancelForegroundDownload(String downloadKey) { + LogUtil.d("%s: CancelForegroundDownload for Uri = %s", TAG, downloadKey); + ListenableFuture<?> unused = + PropagatedFutures.transformAsync( + getInProgressDownloadFuture(downloadKey), + downloadFuture -> { + if (downloadFuture.isPresent()) { + LogUtil.v( + "%s: CancelForegroundDownload future found for key = %s, cancelling...", + TAG, downloadKey); + downloadFuture.get().cancel(false); + } + return immediateVoidFuture(); + }, + sequentialControlExecutor); + } + + private ListenableFuture<Optional<ListenableFuture<Void>>> getInProgressDownloadFuture( + String key) { + return PropagatedFutures.transformAsync( + foregroundDownloadFutureMap.containsKey(key), + isInForeground -> + isInForeground ? foregroundDownloadFutureMap.get(key) : downloadFutureMap.get(key), + sequentialControlExecutor); + } + + private static DownloadFutureMap.StateChangeCallbacks createCallbacksForForegroundService( + Context context, Optional<Class<?>> foregroundDownloadServiceClassOptional) { + return new DownloadFutureMap.StateChangeCallbacks() { + @Override + public void onAdd(String key, int newSize) { + // Only start foreground service if this is the first future we are adding. + if (newSize == 1 && foregroundDownloadServiceClassOptional.isPresent()) { + NotificationUtil.startForegroundDownloadService( + context, foregroundDownloadServiceClassOptional.get(), key); + } + } + + @Override + public void onRemove(String key, int newSize) { + // Only stop foreground service if there are no more futures remaining. + if (newSize == 0 && foregroundDownloadServiceClassOptional.isPresent()) { + NotificationUtil.stopForegroundDownloadService( + context, foregroundDownloadServiceClassOptional.get(), key); + } + } + }; } } diff --git a/java/com/google/android/libraries/mobiledatadownload/lite/annotations/BUILD b/java/com/google/android/libraries/mobiledatadownload/lite/annotations/BUILD index fd00b3b..17ac54b 100644 --- a/java/com/google/android/libraries/mobiledatadownload/lite/annotations/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/lite/annotations/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], diff --git a/java/com/google/android/libraries/mobiledatadownload/logger/BUILD b/java/com/google/android/libraries/mobiledatadownload/logger/BUILD index d8a3560..6395421 100644 --- a/java/com/google/android/libraries/mobiledatadownload/logger/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/logger/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -27,5 +28,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload:Flags", "//java/com/google/android/libraries/mobiledatadownload:Logger", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/logger/FileGroupPopulatorLogger.java b/java/com/google/android/libraries/mobiledatadownload/logger/FileGroupPopulatorLogger.java index 435f3b3..d7597b9 100644 --- a/java/com/google/android/libraries/mobiledatadownload/logger/FileGroupPopulatorLogger.java +++ b/java/com/google/android/libraries/mobiledatadownload/logger/FileGroupPopulatorLogger.java @@ -18,6 +18,7 @@ package com.google.android.libraries.mobiledatadownload.logger; import com.google.android.libraries.mobiledatadownload.Flags; import com.google.android.libraries.mobiledatadownload.Logger; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; +import com.google.mobiledatadownload.LogEnumsProto.MddDownloadResult; /** The event logger for {@code FileGroupPopulator}'s. */ public final class FileGroupPopulatorLogger { @@ -32,21 +33,22 @@ public final class FileGroupPopulatorLogger { /** Logs the refresh result of {@code ManifestFileGroupPopulator}. */ public void logManifestFileGroupPopulatorRefreshResult( - int code, String manifestId, String ownerPackageName, String manifestFileUrl) { + MddDownloadResult.Code code, + String manifestId, + String ownerPackageName, + String manifestFileUrl) { int sampleInterval = flags.mddDefaultSampleInterval(); if (!LogUtil.shouldSampleInterval(sampleInterval)) { return; } - Void logData = null; } /** Logs the refresh result of {@code GellerFileGroupPopulator}. */ public void logGddFileGroupPopulatorRefreshResult( - int code, String configurationId, String ownerPackageName, String corpus) { + MddDownloadResult.Code code, String configurationId, String ownerPackageName, String corpus) { int sampleInterval = flags.mddDefaultSampleInterval(); if (!LogUtil.shouldSampleInterval(sampleInterval)) { return; } - Void logData = null; } } diff --git a/java/com/google/android/libraries/mobiledatadownload/monitor/BUILD b/java/com/google/android/libraries/mobiledatadownload/monitor/BUILD index caeaa3c..2f77809 100644 --- a/java/com/google/android/libraries/mobiledatadownload/monitor/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/monitor/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -27,10 +28,11 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload:TimeSource", "//java/com/google/android/libraries/mobiledatadownload/file/monitors", "//java/com/google/android/libraries/mobiledatadownload/file/spi", + "//java/com/google/android/libraries/mobiledatadownload/internal:AndroidTimeSource", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LoggingStateStore", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", - "//java/com/google/android/libraries/mobiledatadownload/tracing", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "@androidx_annotation_annotation", "@com_google_code_findbugs_jsr305", "@com_google_guava_guava", @@ -45,11 +47,13 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload:TimeSource", "//java/com/google/android/libraries/mobiledatadownload/file/monitors", "//java/com/google/android/libraries/mobiledatadownload/file/spi", + "//java/com/google/android/libraries/mobiledatadownload/internal:AndroidTimeSource", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", "//java/com/google/android/libraries/mobiledatadownload/lite:DownloadListener", "//java/com/google/android/libraries/mobiledatadownload/lite:DownloadProgressMonitor", "@androidx_annotation_annotation", "@com_google_code_findbugs_jsr305", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/monitor/DownloadProgressMonitor.java b/java/com/google/android/libraries/mobiledatadownload/monitor/DownloadProgressMonitor.java index 5dcbf6a..b8e7307 100644 --- a/java/com/google/android/libraries/mobiledatadownload/monitor/DownloadProgressMonitor.java +++ b/java/com/google/android/libraries/mobiledatadownload/monitor/DownloadProgressMonitor.java @@ -24,12 +24,12 @@ import com.google.android.libraries.mobiledatadownload.file.spi.Monitor; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.android.libraries.mobiledatadownload.lite.SingleFileDownloadProgressMonitor; import com.google.common.util.concurrent.MoreExecutors; +import com.google.errorprone.annotations.concurrent.GuardedBy; import java.util.HashMap; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import javax.annotation.Nullable; -import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.ThreadSafe; /** diff --git a/java/com/google/android/libraries/mobiledatadownload/monitor/NetworkUsageMonitor.java b/java/com/google/android/libraries/mobiledatadownload/monitor/NetworkUsageMonitor.java index 413a2d1..d41f45c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/monitor/NetworkUsageMonitor.java +++ b/java/com/google/android/libraries/mobiledatadownload/monitor/NetworkUsageMonitor.java @@ -15,7 +15,6 @@ */ package com.google.android.libraries.mobiledatadownload.monitor; -import static com.google.android.libraries.mobiledatadownload.tracing.TracePropagation.propagateFutureCallback; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static java.util.concurrent.TimeUnit.SECONDS; @@ -31,8 +30,8 @@ import com.google.android.libraries.mobiledatadownload.file.monitors.ByteCountin import com.google.android.libraries.mobiledatadownload.file.spi.Monitor; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.android.libraries.mobiledatadownload.internal.logging.LoggingStateStore; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.mobiledatadownload.internal.MetadataProto.FileGroupLoggingState; import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; @@ -98,6 +97,7 @@ public class NetworkUsageMonitor implements Monitor { * @param uri The Uri of the data file. * @param groupKey The groupKey part of the file group. * @param buildId The build id of the file group. + * @param variantId The variant id of the file group. * @param versionNumber The version number of the file group. * @param loggingStateStore The storage for the network usage logs */ @@ -105,12 +105,14 @@ public class NetworkUsageMonitor implements Monitor { Uri uri, GroupKey groupKey, long buildId, + String variantId, int versionNumber, LoggingStateStore loggingStateStore) { FileGroupLoggingState fileGroupLoggingStateKey = FileGroupLoggingState.newBuilder() .setGroupKey(groupKey) .setBuildId(buildId) + .setVariantId(variantId) .setFileGroupVersionNumber(versionNumber) .build(); @@ -189,26 +191,25 @@ public class NetworkUsageMonitor implements Monitor { .setWifiUsage(wifiCounter.getAndSet(0)) .build()); - Futures.addCallback( + PropagatedFutures.addCallback( incrementDataUsage, - propagateFutureCallback( - new FutureCallback<Void>() { - @Override - public void onSuccess(Void unused) { - LogUtil.d( - "%s: Successfully incremented LoggingStateStore network usage for %s", - TAG, fileGroupLoggingStateKey.getGroupKey().getGroupName()); - } + new FutureCallback<Void>() { + @Override + public void onSuccess(Void unused) { + LogUtil.d( + "%s: Successfully incremented LoggingStateStore network usage for %s", + TAG, fileGroupLoggingStateKey.getGroupKey().getGroupName()); + } - @Override - public void onFailure(Throwable t) { - LogUtil.e( - t, - "%s: Unable to increment LoggingStateStore network usage for %s", - TAG, - fileGroupLoggingStateKey.getGroupKey().getGroupName()); - } - }), + @Override + public void onFailure(Throwable t) { + LogUtil.e( + t, + "%s: Unable to increment LoggingStateStore network usage for %s", + TAG, + fileGroupLoggingStateKey.getGroupKey().getGroupName()); + } + }, directExecutor()); } } diff --git a/java/com/google/android/libraries/mobiledatadownload/populator/BUILD b/java/com/google/android/libraries/mobiledatadownload/populator/BUILD index 42f7e73..d9d2a7a 100644 --- a/java/com/google/android/libraries/mobiledatadownload/populator/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/populator/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -36,6 +37,7 @@ android_library( ":DataFileGroupOverrider", "//java/com/google/android/libraries/mobiledatadownload", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "//proto:download_config_java_proto_lite", "@com_google_guava_guava", ], @@ -81,6 +83,8 @@ android_library( deps = [ ":ManifestConfigOverrider", "//java/com/google/android/libraries/mobiledatadownload", + "//java/com/google/android/libraries/mobiledatadownload:AggregateException", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "//proto:download_config_java_proto_lite", "@androidx_annotation_annotation", @@ -105,7 +109,9 @@ android_library( ":ManifestConfigOverrider", "//java/com/google/android/libraries/mobiledatadownload", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", + "//java/com/google/android/libraries/mobiledatadownload/populator/proto:metadata_java_proto_lite", "//proto:download_config_java_proto_lite", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], ) @@ -131,6 +137,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/tracing", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "//proto:download_config_java_proto_lite", + "//proto:log_enums_java_proto_lite", "@androidx_annotation_annotation", "@com_google_code_findbugs_jsr305", "@com_google_guava_guava", @@ -143,8 +150,6 @@ android_library( srcs = [ "ManifestFileMetadataStore.java", ], - # DO NOT ADD VISIBILITY: this isn't an open interface for clients to implement. - visibility = ["//visibility:private"], deps = [ "//java/com/google/android/libraries/mobiledatadownload/populator/proto:metadata_java_proto_lite", "@com_google_guava_guava", diff --git a/java/com/google/android/libraries/mobiledatadownload/populator/LocaleOverrider.java b/java/com/google/android/libraries/mobiledatadownload/populator/LocaleOverrider.java index 1985caa..1556c9a 100644 --- a/java/com/google/android/libraries/mobiledatadownload/populator/LocaleOverrider.java +++ b/java/com/google/android/libraries/mobiledatadownload/populator/LocaleOverrider.java @@ -28,6 +28,7 @@ import com.google.common.base.Supplier; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; import com.google.mobiledatadownload.DownloadConfigProto.ManifestConfig; import java.util.ArrayList; @@ -68,6 +69,7 @@ public final class LocaleOverrider implements ManifestConfigOverrider { private Executor lightweightExecutor; /** only one of setLocaleSupplier or setLocaleFutureSupplier is required */ + @CanIgnoreReturnValue public Builder setLocaleSupplier(Supplier<Locale> localeSupplier) { this.localeSupplier = () -> Futures.immediateFuture(localeSupplier.get()); this.lightweightExecutor = @@ -75,6 +77,7 @@ public final class LocaleOverrider implements ManifestConfigOverrider { return this; } + @CanIgnoreReturnValue public Builder setLocaleFutureSupplier( Supplier<ListenableFuture<Locale>> localeSupplier, Executor lightweightExecutor) { this.localeSupplier = localeSupplier; @@ -87,6 +90,7 @@ public final class LocaleOverrider implements ManifestConfigOverrider { * the config. The set of Locale should be related to ONE {@code group_name} of {@link * DataFilegroup}. */ + @CanIgnoreReturnValue public Builder setMatchStrategy( BiFunction<Locale, Set<Locale>, Optional<Locale>> matchStrategy) { this.matchStrategy = matchStrategy; diff --git a/java/com/google/android/libraries/mobiledatadownload/populator/ManifestConfigFileParser.java b/java/com/google/android/libraries/mobiledatadownload/populator/ManifestConfigFileParser.java new file mode 100644 index 0000000..61ecba3 --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/populator/ManifestConfigFileParser.java @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.libraries.mobiledatadownload.populator; + +import android.net.Uri; + +import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; +import com.google.android.libraries.mobiledatadownload.populator.ManifestFileGroupPopulator.ManifestConfigParser; +import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; +import com.google.android.libraries.mobiledatadownload.file.openers.ReadProtoOpener; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.mobiledatadownload.DownloadConfigProto.ManifestConfig; + +import java.util.concurrent.Executor; + +/** + * The default manifest parser that parses a given manifest file to {@link ManifestConfig}. + * + * <p>The format of the manifest file supported by this {@link ManifestConfigFileParser} is proto. + * That is, the content of manifest file should be accepted by {@link + * ManifestConfig#parseFrom(byte[])}. + */ +public final class ManifestConfigFileParser implements ManifestConfigParser { + + private static final String TAG = "ManifestConfigFileParser"; + + private final SynchronousFileStorage fileStorage; + private final Executor backgroundExecutor; + + public ManifestConfigFileParser(SynchronousFileStorage fileStorage, Executor backgroundExecutor) { + this.fileStorage = fileStorage; + this.backgroundExecutor = backgroundExecutor; + } + + @Override + public ListenableFuture<ManifestConfig> parse(Uri fileUri) { + return PropagatedFutures.submit( + () -> { + LogUtil.d("%s: Start parsing manifest file at %s", TAG, fileUri); + ManifestConfig manifestConfig = + fileStorage.open(fileUri, ReadProtoOpener.create(ManifestConfig.parser())); + return manifestConfig; + }, + backgroundExecutor); + } +}
\ No newline at end of file diff --git a/java/com/google/android/libraries/mobiledatadownload/populator/ManifestConfigFlagPopulator.java b/java/com/google/android/libraries/mobiledatadownload/populator/ManifestConfigFlagPopulator.java index 37ffbc7..9169d17 100644 --- a/java/com/google/android/libraries/mobiledatadownload/populator/ManifestConfigFlagPopulator.java +++ b/java/com/google/android/libraries/mobiledatadownload/populator/ManifestConfigFlagPopulator.java @@ -23,8 +23,10 @@ import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.common.base.Joiner; import com.google.common.base.Optional; import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.util.concurrent.ListenableFuture; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; import com.google.mobiledatadownload.DownloadConfigProto.ManifestConfig; @@ -56,6 +58,7 @@ public final class ManifestConfigFlagPopulator implements FileGroupPopulator { private Optional<ManifestConfigOverrider> overriderOptional = Optional.absent(); /** Set the ManifestConfig supplier. */ + @CanIgnoreReturnValue public Builder setManifestConfigSupplier(Supplier<ManifestConfig> manifestConfigSupplier) { this.manifestConfigSupplier = manifestConfigSupplier; return this; @@ -65,6 +68,7 @@ public final class ManifestConfigFlagPopulator implements FileGroupPopulator { * Sets the optional Overrider that takes a {@link ManifestConfig} and returns a list of {@link * DataFileGroup} which will be added to MDD. The Overrider will enable the on device targeting. */ + @CanIgnoreReturnValue public Builder setOverriderOptional(Optional<ManifestConfigOverrider> overriderOptional) { this.overriderOptional = overriderOptional; return this; @@ -104,6 +108,10 @@ public final class ManifestConfigFlagPopulator implements FileGroupPopulator { LogUtil.d("%s: Add groups [%s] from ManifestConfig to MDD.", TAG, groups); return ManifestConfigHelper.refreshFromManifestConfig( - mobileDataDownload, manifestConfigSupplier.get(), overriderOptional); + mobileDataDownload, + manifestConfigSupplier.get(), + overriderOptional, + /* accounts= */ ImmutableList.of(), + /* addGroupsWithVariantId= */ false); } } diff --git a/java/com/google/android/libraries/mobiledatadownload/populator/ManifestConfigHelper.java b/java/com/google/android/libraries/mobiledatadownload/populator/ManifestConfigHelper.java index d2c8722..eb74c8a 100644 --- a/java/com/google/android/libraries/mobiledatadownload/populator/ManifestConfigHelper.java +++ b/java/com/google/android/libraries/mobiledatadownload/populator/ManifestConfigHelper.java @@ -15,9 +15,11 @@ */ package com.google.android.libraries.mobiledatadownload.populator; -import android.util.Log; +import android.accounts.Account; import com.google.android.libraries.mobiledatadownload.AddFileGroupRequest; +import com.google.android.libraries.mobiledatadownload.AggregateException; import com.google.android.libraries.mobiledatadownload.MobileDataDownload; +import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentFuture; import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Optional; @@ -40,69 +42,150 @@ public final class ManifestConfigHelper { private final MobileDataDownload mobileDataDownload; private final Optional<ManifestConfigOverrider> overriderOptional; + private final List<Account> accounts; + private final boolean addGroupsWithVariantId; /** Creates a new helper for converting manifest configs into data file groups. */ ManifestConfigHelper( - MobileDataDownload mobileDataDownload, Optional<ManifestConfigOverrider> overriderOptional) { + MobileDataDownload mobileDataDownload, + Optional<ManifestConfigOverrider> overriderOptional, + List<Account> accounts, + boolean addGroupsWithVariantId) { this.mobileDataDownload = mobileDataDownload; this.overriderOptional = overriderOptional; + this.accounts = accounts; + this.addGroupsWithVariantId = addGroupsWithVariantId; } /** * Reads file groups from {@link ManifestConfig} and adds to MDD after applying the {@link - * ManifestConfigOverrider} if it's present. This static method is shared with {@link - * ManifestFileGroupPopulator}. + * ManifestConfigOverrider} if it's present. + * + * <p>This static method encapsulates shared logic between a few populators: + * + * <ul> + * <li>{@link ManifestFileGroupPopulator} + * <li>{@link ManifestConfigFlagPopulator} + * <li>{@link LocalManifestFileGroupPopulator} + * <li>{@link EmbeddedAssetManifestPopulator} + * </ul> * - * @param mobileDataDownload The MDD instance. - * @param manifestConfig The proto that contains configs for file groups and modifiers. + * @param mobileDataDownload The MDD instance + * @param manifestConfig The proto that contains configs for file groups and modifiers * @param overriderOptional An optional overrider that takes manifest config and returns a list of - * file groups to be added to MDD. + * file groups to be added ot MDD + * @param accounts A list of accounts that the parsed file groups should be associated with + * @param addGroupsWithVariantId whether variantId should be included when adding the parsed file + * groups */ static ListenableFuture<Void> refreshFromManifestConfig( MobileDataDownload mobileDataDownload, ManifestConfig manifestConfig, - Optional<ManifestConfigOverrider> overriderOptional) { - ManifestConfigHelper helper = new ManifestConfigHelper(mobileDataDownload, overriderOptional); + Optional<ManifestConfigOverrider> overriderOptional, + List<Account> accounts, + boolean addGroupsWithVariantId) { + ManifestConfigHelper helper = + new ManifestConfigHelper( + mobileDataDownload, overriderOptional, accounts, addGroupsWithVariantId); return PropagatedFluentFuture.from(helper.applyOverrider(manifestConfig)) - .transformAsync(helper::addAllFileGroups, MoreExecutors.directExecutor()); + .transformAsync(helper::addAllFileGroups, MoreExecutors.directExecutor()) + .catchingAsync( + AggregateException.class, + ex -> Futures.immediateVoidFuture(), + MoreExecutors.directExecutor()); } /** Adds the specified list of file groups to MDD. */ ListenableFuture<Void> addAllFileGroups(List<DataFileGroup> fileGroups) { List<ListenableFuture<Boolean>> addFileGroupFutures = new ArrayList<>(); + Optional<String> variantId = Optional.absent(); for (DataFileGroup dataFileGroup : fileGroups) { if (dataFileGroup == null || dataFileGroup.getGroupName().isEmpty()) { continue; } - ListenableFuture<Boolean> addFileGroupFuture = - mobileDataDownload.addFileGroup( - AddFileGroupRequest.newBuilder().setDataFileGroup(dataFileGroup).build()); + // Include variantId if variant is present and helper is configured to do so + if (addGroupsWithVariantId && !dataFileGroup.getVariantId().isEmpty()) { + variantId = Optional.of(dataFileGroup.getVariantId()); + } - PropagatedFutures.addCallback( - addFileGroupFuture, - new FutureCallback<Boolean>() { - @Override - public void onSuccess(Boolean result) { - String groupName = dataFileGroup.getGroupName(); - if (result.booleanValue()) { - Log.d(TAG, "Added file groups " + groupName); - } else { - Log.d(TAG, "Failed to add file group " + groupName); - } - } + AddFileGroupRequest.Builder addFileGroupRequestBuilder = + AddFileGroupRequest.newBuilder() + .setDataFileGroup(dataFileGroup) + .setVariantIdOptional(variantId); - @Override - public void onFailure(Throwable t) { - Log.e(TAG, "Failed to add file group", t); - } - }, - MoreExecutors.directExecutor()); + // Add once without any account + ListenableFuture<Boolean> addFileGroupFuture = + mobileDataDownload.addFileGroup(addFileGroupRequestBuilder.build()); + attachLoggingCallback( + addFileGroupFuture, + dataFileGroup.getGroupName(), + /* account= */ Optional.absent(), + variantId); addFileGroupFutures.add(addFileGroupFuture); + + // Add for each account + for (Account account : accounts) { + ListenableFuture<Boolean> addFileGroupFutureWithAccount = + mobileDataDownload.addFileGroup( + addFileGroupRequestBuilder.setAccountOptional(Optional.of(account)).build()); + attachLoggingCallback( + addFileGroupFutureWithAccount, + dataFileGroup.getGroupName(), + Optional.of(account), + variantId); + addFileGroupFutures.add(addFileGroupFutureWithAccount); + } } return PropagatedFutures.whenAllComplete(addFileGroupFutures) - .call(() -> null, MoreExecutors.directExecutor()); + .call( + () -> { + AggregateException.throwIfFailed(addFileGroupFutures, "Failed to add file groups"); + return null; + }, + MoreExecutors.directExecutor()); + } + + private void attachLoggingCallback( + ListenableFuture<Boolean> addFileGroupFuture, + String groupName, + Optional<Account> account, + Optional<String> variant) { + PropagatedFutures.addCallback( + addFileGroupFuture, + new FutureCallback<Boolean>() { + @Override + public void onSuccess(Boolean result) { + if (result.booleanValue()) { + LogUtil.d( + "%s: Added file group %s with account: %s, variant: %s", + TAG, + groupName, + String.valueOf(account.orNull()), + String.valueOf(variant.orNull())); + } else { + LogUtil.d( + "%s: Failed to add file group %s with account: %s, variant: %s", + TAG, + groupName, + String.valueOf(account.orNull()), + String.valueOf(variant.orNull())); + } + } + + @Override + public void onFailure(Throwable t) { + LogUtil.e( + t, + "%s: Failed to add file group %s with account: %s, variant: %s", + TAG, + groupName, + String.valueOf(account.orNull()), + String.valueOf(variant.orNull())); + } + }, + MoreExecutors.directExecutor()); } /** Applies the overrider to the manifest config to generate a list of file groups for adding. */ diff --git a/java/com/google/android/libraries/mobiledatadownload/populator/ManifestFileGroupPopulator.java b/java/com/google/android/libraries/mobiledatadownload/populator/ManifestFileGroupPopulator.java index 66d26ab..b8d3551 100644 --- a/java/com/google/android/libraries/mobiledatadownload/populator/ManifestFileGroupPopulator.java +++ b/java/com/google/android/libraries/mobiledatadownload/populator/ManifestFileGroupPopulator.java @@ -22,8 +22,6 @@ import static com.google.common.util.concurrent.Futures.immediateVoidFuture; import android.content.Context; import android.net.Uri; import androidx.annotation.VisibleForTesting; -import com.google.mobiledatadownload.populator.MetadataProto.ManifestFileBookkeeping; -import com.google.mobiledatadownload.populator.MetadataProto.ManifestFileBookkeeping.Status; import com.google.android.libraries.mobiledatadownload.AggregateException; import com.google.android.libraries.mobiledatadownload.DownloadException; import com.google.android.libraries.mobiledatadownload.DownloadException.DownloadResultCode; @@ -40,6 +38,7 @@ import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStora import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.android.libraries.mobiledatadownload.internal.util.DirectoryUtil; import com.google.android.libraries.mobiledatadownload.logger.FileGroupPopulatorLogger; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedExecutionSequencer; import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentFuture; import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Optional; @@ -49,9 +48,13 @@ import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ExecutionSequencer; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.ListenableFuture; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; import com.google.mobiledatadownload.DownloadConfigProto.ManifestConfig; import com.google.mobiledatadownload.DownloadConfigProto.ManifestFileFlag; +import com.google.mobiledatadownload.LogEnumsProto.MddDownloadResult; +import com.google.mobiledatadownload.populator.MetadataProto.ManifestFileBookkeeping; +import com.google.mobiledatadownload.populator.MetadataProto.ManifestFileBookkeeping.Status; import java.io.IOException; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicReference; @@ -91,8 +94,7 @@ import javax.inject.Singleton; * hosting service needs to support ETag (e.g. Lorry), otherwise the behavior will be unexpected. * Talk to <internal>@ if you are not sure if the hosting service supports ETag. * - * <p>Note that {@link SynchronousFileStorage} and {@link ProtoDataStoreFactory} passed to builder - * must be @Singleton. + * <p> * * <p>This class is @Singleton, because it provides the guarantee that all the operations are * serialized correctly by {@link ExecutionSequencer}. @@ -109,10 +111,16 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { ListenableFuture<ManifestConfig> parse(Uri fileUri); } + /** Client-provided supplier of a condition whether the populator should be enabled. */ + public interface EnabledSupplier { + boolean isEnabled(); + } + /** Builder for {@link ManifestFileGroupPopulator}. */ public static final class Builder { private boolean allowsInsecureHttp = false; private boolean dedupDownloadWithEtag = true; + private boolean forceManifestSyncs = true; private Context context; private Supplier<ManifestFileFlag> manifestFileFlagSupplier; private Supplier<FileDownloader> fileDownloader; @@ -124,12 +132,15 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { private Optional<ManifestConfigOverrider> overriderOptional = Optional.absent(); private Optional<String> instanceIdOptional = Optional.absent(); private Flags flags = new Flags() {}; + // Enabled the populator if no EnabledSupplier is provided. + private EnabledSupplier enabledSupplier = () -> true; /** * Sets the flag that allows insecure http. * * <p>For testing only. */ + @CanIgnoreReturnValue @VisibleForTesting Builder setAllowsInsecureHttp(boolean allowsInsecureHttp) { this.allowsInsecureHttp = allowsInsecureHttp; @@ -140,18 +151,41 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { * By default, an HTTP HEAD request is made to avoid duplicate downloads of the manifest file. * Setting this to false disables that behavior. */ + @CanIgnoreReturnValue public Builder setDedupDownloadWithEtag(boolean dedup) { this.dedupDownloadWithEtag = dedup; return this; } + /** + * Force manifest syncs when {@link setDedupDownloadWithEtag} is set to false. + * + * <p>When NOT deduping with ETag, it's possible that a downloaded version of a manifest may + * override a potentially newer version of a manifest, preventing new file groups from being + * synced. + * + * <p>This flag controls whether or not the fix (always downloading the manifest) should be + * used. + * + * <p>NOTE: By default, this flag will be set to true -- if clients would rather have a + * controlled rollout of this behavior change, they should include this option in their builder + * and connect this to an experimental rollout system. See b/243926815 for more details. + */ + @CanIgnoreReturnValue + public Builder setForceManifestSyncsWithoutETag(boolean forceManifestSyncs) { + this.forceManifestSyncs = forceManifestSyncs; + return this; + } + /** Sets the context. */ + @CanIgnoreReturnValue public Builder setContext(Context context) { this.context = context.getApplicationContext(); return this; } /** Sets the manifest file flag. */ + @CanIgnoreReturnValue public Builder setManifestFileFlagSupplier( Supplier<ManifestFileFlag> manifestFileFlagSupplier) { this.manifestFileFlagSupplier = manifestFileFlagSupplier; @@ -159,58 +193,80 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { } /** Sets the file downloader. */ + @CanIgnoreReturnValue public Builder setFileDownloader(Supplier<FileDownloader> fileDownloader) { this.fileDownloader = fileDownloader; return this; } /** Sets the manifest config parser that takes file uri and returns {@link ManifestConfig}. */ + @CanIgnoreReturnValue public Builder setManifestConfigParser(ManifestConfigParser manifestConfigParser) { this.manifestConfigParser = manifestConfigParser; return this; } /** Sets the mobstore file storage. Mobstore file storage must be singleton. */ + @CanIgnoreReturnValue public Builder setFileStorage(SynchronousFileStorage fileStorage) { this.fileStorage = fileStorage; return this; } /** Sets the background executor that executes populator's tasks sequentially. */ + @CanIgnoreReturnValue public Builder setBackgroundExecutor(Executor backgroundExecutor) { this.backgroundExecutor = backgroundExecutor; return this; } - /** Sets the ManifestFileMetadataStore. */ + /** + * Sets the ManifestFileMetadataStore. + * + * <p> + */ + @CanIgnoreReturnValue public Builder setMetadataStore(ManifestFileMetadataStore manifestFileMetadataStore) { this.manifestFileMetadataStore = manifestFileMetadataStore; return this; } /** Sets the MDD logger. */ + @CanIgnoreReturnValue public Builder setLogger(Logger logger) { this.logger = logger; return this; } /** Sets the optional manifest config overrider. */ + @CanIgnoreReturnValue public Builder setOverriderOptional(Optional<ManifestConfigOverrider> overriderOptional) { this.overriderOptional = overriderOptional; return this; } /** Sets the optional instance ID. */ + @CanIgnoreReturnValue public Builder setInstanceIdOptional(Optional<String> instanceIdOptional) { this.instanceIdOptional = instanceIdOptional; return this; } + @CanIgnoreReturnValue public Builder setFlags(Flags flags) { this.flags = flags; return this; } + /** + * Sets the condition to check whether the populator should be enabled. If the value, returned + * by the condition is {@code false}, {@code refreshFileGroups} should do nothing. + */ + public Builder setEnabledSupplier(EnabledSupplier enabledSupplier) { + this.enabledSupplier = enabledSupplier; + return this; + } + public ManifestFileGroupPopulator build() { Preconditions.checkNotNull(context, "Must call setContext() before build()."); Preconditions.checkNotNull( @@ -230,6 +286,7 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { private final boolean allowsInsecureHttp; private final boolean dedupDownloadWithEtag; + private final boolean forceManifestSyncs; private final Context context; private final Uri manifestDirectoryUri; private final Supplier<ManifestFileFlag> manifestFileFlagSupplier; @@ -241,7 +298,10 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { private final ManifestFileMetadataStore manifestFileMetadataStore; private final FileGroupPopulatorLogger eventLogger; // We use futureSerializer for synchronization. - private final ExecutionSequencer futureSerializer = ExecutionSequencer.create(); + private final PropagatedExecutionSequencer futureSerializer = + PropagatedExecutionSequencer.create(); + private final EnabledSupplier enabledSupplier; + /** Returns a Builder for {@link ManifestFileGroupPopulator}. */ public static Builder builder() { @@ -251,6 +311,7 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { private ManifestFileGroupPopulator(Builder builder) { this.allowsInsecureHttp = builder.allowsInsecureHttp; this.dedupDownloadWithEtag = builder.dedupDownloadWithEtag; + this.forceManifestSyncs = builder.forceManifestSyncs; this.context = builder.context; this.manifestDirectoryUri = DirectoryUtil.getManifestDirectory(builder.context, builder.instanceIdOptional); @@ -262,6 +323,7 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { this.overriderOptional = builder.overriderOptional; this.eventLogger = new FileGroupPopulatorLogger(builder.logger, builder.flags); this.manifestFileMetadataStore = builder.manifestFileMetadataStore; + this.enabledSupplier = builder.enabledSupplier; } @Override @@ -277,7 +339,8 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { if (manifestFileFlag == null || manifestFileFlag.equals(ManifestFileFlag.getDefaultInstance())) { LogUtil.w("%s: The ManifestFileFlag is empty.", TAG); - logRefreshResult(0, ManifestFileFlag.getDefaultInstance()); + logRefreshResult( + MddDownloadResult.Code.SUCCESS, ManifestFileFlag.getDefaultInstance()); return immediateVoidFuture(); } @@ -288,9 +351,15 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { private ListenableFuture<Void> refreshFileGroups( MobileDataDownload mobileDataDownload, ManifestFileFlag manifestFileFlag) { + if(!enabledSupplier.isEnabled()){ + LogUtil.d("%s: The populator was disabled by enabledSupplier", TAG); + return immediateVoidFuture(); + } if (!validate(manifestFileFlag)) { - logRefreshResult(0, manifestFileFlag); + logRefreshResult( + MddDownloadResult.Code.MANIFEST_FILE_GROUP_POPULATOR_INVALID_FLAG_ERROR, + manifestFileFlag); LogUtil.e("%s: Invalid manifest config from manifest flag.", TAG); return immediateFailedFuture(new IllegalArgumentException("Invalid manifest flag.")); } @@ -402,7 +471,7 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { manifestFileFlag); // If there is any failure, it should have been thrown already. Therefore, we log refresh // success here. - logRefreshResult(0, manifestFileFlag); + logRefreshResult(MddDownloadResult.Code.SUCCESS, manifestFileFlag); return immediateVoidFuture(); }, backgroundExecutor); @@ -430,7 +499,11 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { .transformAsync( (final ManifestConfig manifestConfig) -> ManifestConfigHelper.refreshFromManifestConfig( - mobileDataDownload, manifestConfig, overriderOptional), + mobileDataDownload, + manifestConfig, + overriderOptional, + /* accounts= */ ImmutableList.of(), + /* addGroupsWithVariantId= */ false), backgroundExecutor) .transformAsync( voidArg -> { @@ -481,7 +554,7 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { LogUtil.d("%s: Prepare for downloading manifest file.", TAG); if (!dedupDownloadWithEtag) { - return immediateVoidFuture(); + return handleManifestDedupWithoutETag(urlToDownload, manifestFileUri, bookkeepingRef); } ManifestFileBookkeeping bookkeeping = bookkeepingRef.get(); @@ -524,6 +597,41 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { backgroundExecutor); } + /** + * Handle Manifest Bookkeeping when ETag check should be bypassed. + * + * <p>If forced syncs are enabled, the existing manifest file will be deleted and the bookkeeping + * reference will be updated to a default value. This forces the manifest to be redownloaded. + * + * <p>If forced syncs are disabled, this is a no-op and existing bookkeeping will be used. This + * reuses a downloaded manifest if one exists, or continues a download of a pending manifest. + */ + private ListenableFuture<Void> handleManifestDedupWithoutETag( + String urlToDownload, + Uri manifestFileUri, + AtomicReference<ManifestFileBookkeeping> bookkeepingRef) { + LogUtil.d( + "%s: Not relying on etag to dedup manifest -- checking if manifest should be force" + + " downloaded", + TAG); + if (forceManifestSyncs) { + LogUtil.d( + "%s: forcing re-download; urlToDownload = %s;" + " manifestFileUri = %s", + TAG, urlToDownload, manifestFileUri); + try { + deleteManifestFileChecked(manifestFileUri); + } catch (DownloadException e) { + return immediateFailedFuture(e); + } + bookkeepingRef.set(createDefaultManifestFileBookkeeping(urlToDownload)); + } else { + LogUtil.d( + "%s: not forcing re-download; urlToDownload = %s;" + " manifestFileUri =%s", + TAG, urlToDownload, manifestFileUri); + } + return immediateVoidFuture(); + } + private ListenableFuture<Void> checkForContentChangeAfterDownload( String urlToDownload, Uri manifestFileUri, @@ -531,6 +639,10 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { LogUtil.d("%s: Finalize for downloading manifest file.", TAG); if (!dedupDownloadWithEtag) { + LogUtil.d( + "%s: Not relying on etag to dedup manifest, so the downloaded manifest is" + + " assumed to be the latest; urlToDownload = %s, manifestFileUri = %s", + TAG, urlToDownload, manifestFileUri); return immediateVoidFuture(); } @@ -610,15 +722,17 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { } } + // incompatible argument for parameter code of logManifestFileGroupPopulatorRefreshResult. + @SuppressWarnings("nullness:argument.type.incompatible") private void logRefreshResult(DownloadException e, ManifestFileFlag manifestFileFlag) { eventLogger.logManifestFileGroupPopulatorRefreshResult( - 0, + MddDownloadResult.Code.forNumber(e.getDownloadResultCode().getCode()), manifestFileFlag.getManifestId(), context.getPackageName(), manifestFileFlag.getManifestFileUrl()); } - private void logRefreshResult(int code, ManifestFileFlag manifestFileFlag) { + private void logRefreshResult(MddDownloadResult.Code code, ManifestFileFlag manifestFileFlag) { eventLogger.logManifestFileGroupPopulatorRefreshResult( code, manifestFileFlag.getManifestId(), @@ -659,7 +773,7 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { private static ManifestFileBookkeeping createDefaultManifestFileBookkeeping( String manifestFileUrl) { return createManifestFileBookkeeping( - manifestFileUrl, Status.PENDING, /* eTagOptional = */ Optional.absent()); + manifestFileUrl, Status.PENDING, /* eTagOptional= */ Optional.absent()); } private static ManifestFileBookkeeping createManifestFileBookkeeping( diff --git a/java/com/google/android/libraries/mobiledatadownload/populator/ManifestFileMetadataStore.java b/java/com/google/android/libraries/mobiledatadownload/populator/ManifestFileMetadataStore.java index 4d80080..874571b 100644 --- a/java/com/google/android/libraries/mobiledatadownload/populator/ManifestFileMetadataStore.java +++ b/java/com/google/android/libraries/mobiledatadownload/populator/ManifestFileMetadataStore.java @@ -15,9 +15,9 @@ */ package com.google.android.libraries.mobiledatadownload.populator; -import com.google.mobiledatadownload.populator.MetadataProto.ManifestFileBookkeeping; import com.google.common.base.Optional; import com.google.common.util.concurrent.ListenableFuture; +import com.google.mobiledatadownload.populator.MetadataProto.ManifestFileBookkeeping; /** Storage mechanism for ManifestFileBookkeeping. */ interface ManifestFileMetadataStore { diff --git a/java/com/google/android/libraries/mobiledatadownload/populator/SharedPreferencesManifestFileMetadata.java b/java/com/google/android/libraries/mobiledatadownload/populator/SharedPreferencesManifestFileMetadata.java index 8656e91..3fb1db2 100644 --- a/java/com/google/android/libraries/mobiledatadownload/populator/SharedPreferencesManifestFileMetadata.java +++ b/java/com/google/android/libraries/mobiledatadownload/populator/SharedPreferencesManifestFileMetadata.java @@ -17,13 +17,13 @@ package com.google.android.libraries.mobiledatadownload.populator; import android.content.Context; import android.content.SharedPreferences; -import com.google.mobiledatadownload.populator.MetadataProto.ManifestFileBookkeeping; import com.google.android.libraries.mobiledatadownload.internal.util.SharedPreferencesUtil; import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Optional; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.google.common.util.concurrent.ListenableFuture; +import com.google.mobiledatadownload.populator.MetadataProto.ManifestFileBookkeeping; import java.io.IOException; import java.util.concurrent.Executor; diff --git a/java/com/google/android/libraries/mobiledatadownload/populator/SingleDataFileGroupPopulator.java b/java/com/google/android/libraries/mobiledatadownload/populator/SingleDataFileGroupPopulator.java index 10bd8c8..d513168 100644 --- a/java/com/google/android/libraries/mobiledatadownload/populator/SingleDataFileGroupPopulator.java +++ b/java/com/google/android/libraries/mobiledatadownload/populator/SingleDataFileGroupPopulator.java @@ -15,17 +15,20 @@ */ package com.google.android.libraries.mobiledatadownload.populator; +import static com.google.common.util.concurrent.Futures.immediateFuture; + import android.util.Log; import com.google.android.libraries.mobiledatadownload.AddFileGroupRequest; import com.google.android.libraries.mobiledatadownload.FileGroupPopulator; import com.google.android.libraries.mobiledatadownload.MobileDataDownload; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Optional; import com.google.common.base.Supplier; import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; /** @@ -46,6 +49,7 @@ public final class SingleDataFileGroupPopulator implements FileGroupPopulator { private Supplier<DataFileGroup> dataFileGroupSupplier; private Optional<DataFileGroupOverrider> overriderOptional = Optional.absent(); + @CanIgnoreReturnValue public Builder setDataFileGroupSupplier(Supplier<DataFileGroup> dataFileGroupSupplier) { this.dataFileGroupSupplier = dataFileGroupSupplier; return this; @@ -56,6 +60,7 @@ public final class SingleDataFileGroupPopulator implements FileGroupPopulator { * {@link DataFileGroup} after being overridden. If the overrider returns a null data file * group, nothing will be populated. */ + @CanIgnoreReturnValue public Builder setOverriderOptional(Optional<DataFileGroupOverrider> overriderOptional) { this.overriderOptional = overriderOptional; return this; @@ -86,17 +91,17 @@ public final class SingleDataFileGroupPopulator implements FileGroupPopulator { // Override data file group if the overrider is present. If the overrider returns an absent // data file group, nothing will be populated. ListenableFuture<Optional<DataFileGroup>> dataFileGroupOptionalFuture = - Futures.immediateFuture(Optional.absent()); + immediateFuture(Optional.absent()); if (dataFileGroupSupplier.get() != null && !dataFileGroupSupplier.get().getGroupName().isEmpty()) { dataFileGroupOptionalFuture = overriderOptional.isPresent() ? overriderOptional.get().override(dataFileGroupSupplier.get()) - : Futures.immediateFuture(Optional.of(dataFileGroupSupplier.get())); + : immediateFuture(Optional.of(dataFileGroupSupplier.get())); } ListenableFuture<Boolean> addFileGroupFuture = - Futures.transformAsync( + PropagatedFutures.transformAsync( dataFileGroupOptionalFuture, dataFileGroupOptional -> { if (dataFileGroupOptional.isPresent() @@ -107,11 +112,11 @@ public final class SingleDataFileGroupPopulator implements FileGroupPopulator { .build()); } LogUtil.d("%s: Not adding file group because of overrider.", TAG); - return Futures.immediateFuture(false); + return immediateFuture(false); }, MoreExecutors.directExecutor()); - Futures.addCallback( + PropagatedFutures.addCallback( addFileGroupFuture, new FutureCallback<Boolean>() { @Override @@ -131,7 +136,7 @@ public final class SingleDataFileGroupPopulator implements FileGroupPopulator { }, MoreExecutors.directExecutor()); - return Futures.whenAllComplete(addFileGroupFuture) + return PropagatedFutures.whenAllComplete(addFileGroupFuture) .call(() -> null, MoreExecutors.directExecutor()); } } diff --git a/java/com/google/android/libraries/mobiledatadownload/populator/proto/Android.bp b/java/com/google/android/libraries/mobiledatadownload/populator/proto/Android.bp index db31e28..1110b02 100644 --- a/java/com/google/android/libraries/mobiledatadownload/populator/proto/Android.bp +++ b/java/com/google/android/libraries/mobiledatadownload/populator/proto/Android.bp @@ -44,5 +44,7 @@ java_library { apex_available: [ "//apex_available:platform", "com.android.adservices", + "com.android.extservices", + "com.android.ondevicepersonalization", ], } diff --git a/java/com/google/android/libraries/mobiledatadownload/populator/proto/BUILD b/java/com/google/android/libraries/mobiledatadownload/populator/proto/BUILD index 91be276..637afee 100644 --- a/java/com/google/android/libraries/mobiledatadownload/populator/proto/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/populator/proto/BUILD @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//:__subpackages__", ], diff --git a/java/com/google/android/libraries/mobiledatadownload/testing/BUILD b/java/com/google/android/libraries/mobiledatadownload/testing/BUILD new file mode 100644 index 0000000..80f6902 --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/testing/BUILD @@ -0,0 +1,20 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +package( + default_applicable_licenses = ["//:license"], + default_visibility = [ + "//visibility:public", + ], + licenses = ["notice"], +) diff --git a/java/com/google/android/libraries/mobiledatadownload/tracing/BUILD b/java/com/google/android/libraries/mobiledatadownload/tracing/BUILD index 18ae88e..8629dc4 100644 --- a/java/com/google/android/libraries/mobiledatadownload/tracing/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/tracing/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -32,6 +33,7 @@ android_library( android_library( name = "concurrent", srcs = [ + "PropagatedExecutionSequencer.java", "PropagatedFluentFuture.java", "PropagatedFluentFutures.java", "PropagatedFutures.java", diff --git a/java/com/google/android/libraries/mobiledatadownload/tracing/PropagatedExecutionSequencer.java b/java/com/google/android/libraries/mobiledatadownload/tracing/PropagatedExecutionSequencer.java new file mode 100644 index 0000000..0d2073f --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/tracing/PropagatedExecutionSequencer.java @@ -0,0 +1,48 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.tracing; + +import com.google.common.util.concurrent.AsyncCallable; +import com.google.common.util.concurrent.ExecutionSequencer; +import com.google.common.util.concurrent.ListenableFuture; +import java.util.concurrent.Callable; +import java.util.concurrent.Executor; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** Wrapper around {@link ExecutionSequencer} with trace propagation. */ +public final class PropagatedExecutionSequencer { + + private final ExecutionSequencer executionSequencer = ExecutionSequencer.create(); + + private PropagatedExecutionSequencer() {} + + /** Creates a new instance. */ + public static PropagatedExecutionSequencer create() { + return new PropagatedExecutionSequencer(); + } + + /** See {@link ExecutionSequencer#submit(Callable, Executor)}. */ + public <T extends @Nullable Object> ListenableFuture<T> submit( + Callable<T> callable, Executor executor) { + return executionSequencer.submit(callable, executor); + } + + /** See {@link ExecutionSequencer#submitAsync(AsyncCallable, Executor)}. */ + public <T extends @Nullable Object> ListenableFuture<T> submitAsync( + AsyncCallable<T> callable, Executor executor) { + return executionSequencer.submitAsync(callable, executor); + } +} diff --git a/java/com/google/android/libraries/mobiledatadownload/tracing/TracePropagation.java b/java/com/google/android/libraries/mobiledatadownload/tracing/TracePropagation.java index 7c1ee13..d2c9f79 100644 --- a/java/com/google/android/libraries/mobiledatadownload/tracing/TracePropagation.java +++ b/java/com/google/android/libraries/mobiledatadownload/tracing/TracePropagation.java @@ -61,5 +61,10 @@ public final class TracePropagation { return closingFunction; } + @CheckReturnValue + public static Runnable propagateRunnable(Runnable runnable) { + return runnable; + } + private TracePropagation() {} } diff --git a/javatests/Android.bp b/javatests/Android.bp new file mode 100644 index 0000000..49e4e97 --- /dev/null +++ b/javatests/Android.bp @@ -0,0 +1,68 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + + +//########################################################### +// Robolectric test target for testing mdd test lib classes # +//########################################################### +android_app { + name: "MobileDataDownloadPlaceHolderApp", + manifest: "com/google/android/libraries/mobiledatadownload/internal/AndroidManifest.xml", + platform_apis: true, +} + +android_robolectric_test { + + name: "MobileDataDownloadRoboTests", + + srcs: [ + "com/google/android/libraries/mobiledatadownload/internal/*.java", + ], + + exclude_srcs: [ + // Already compiled from mdd-robolectric-library + "com/google/android/libraries/mobiledatadownload/internal/MddTestUtil.java", + // Tests that are not yet ready to be included. + // TODO: (b/256877824) To be removed once the dependency for LabsFutures and ProtoParsers is resolved. + "com/google/android/libraries/mobiledatadownload/internal/MobileDataDownloadManagerTest.java", // Missing LabsFutures + "com/google/android/libraries/mobiledatadownload/internal/FileGroupManagerTest.java", // Missing LabsFutures + "com/google/android/libraries/mobiledatadownload/internal/util/ProtoConversionUtilTest.java", // Missing ProtoParsers + "com/google/android/libraries/mobiledatadownload/internal/MddIsolatedStructuresTest.java", //android.os.symlink and android.os.readlink do not work with robolectric + "com/google/android/libraries/mobiledatadownload/testing/FakeMobileDataDownload.java", // Missing GoogleLogger + "com/google/android/libraries/mobiledatadownload/testing/MddTestDependencies.java", // Missing BaseFileDownloaderModule + "com/google/android/libraries/mobiledatadownload/internal/ExpirationHandlerTest.java" // Test failed + + ], + + java_resource_dirs: ["config"], + + libs: [ + // This jar should not be included, android_robolectric_test soong tasks either ads + // "Robolectric_all-target" or "Robolectric_all-target_upstream" based on the "upstream" + // flag below. + "androidx.test.core", + "mobile_data_downloader_lib", + "mdd-robolectric-library", + ], + + // use external/robolectric, rather than the outdated external/robolectric-shadows. + upstream: true, + + instrumentation_for: "MobileDataDownloadPlaceHolderApp", + +} diff --git a/javatests/com/google/android/libraries/mobiledatadownload/AndroidManifest.xml b/javatests/com/google/android/libraries/mobiledatadownload/AndroidManifest.xml index 945f71c..e89d82f 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/AndroidManifest.xml +++ b/javatests/com/google/android/libraries/mobiledatadownload/AndroidManifest.xml @@ -15,7 +15,6 @@ * limitations under the License. */ --> -<?xml version="1.0" encoding="utf-8"?> <manifest xmlns:android="http://schemas.android.com/apk/res/android" package="com.google.android.libraries.mobiledatadownload" > diff --git a/javatests/com/google/android/libraries/mobiledatadownload/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/BUILD index c80ff58..be40cf1 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/BUILD +++ b/javatests/com/google/android/libraries/mobiledatadownload/BUILD @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. load("//tools/build_defs/testing:bzl_library.bzl", "bzl_library") -load("//javatests/com/google/android/libraries/mobiledatadownload:test_defs.bzl", "mdd_android_test", "mdd_local_test") +load("//javatests/com/google/android/libraries/mobiledatadownload:test_defs.bzl", "PARAMETERIZED_EMULATOR_IMAGES", "mdd_android_test", "mdd_local_test") load("@build_bazel_rules_android//android:rules.bzl", "android_local_test") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -35,14 +36,22 @@ mdd_local_test( "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/file/backends:android", "//java/com/google/android/libraries/mobiledatadownload/file/openers:stream", + "//java/com/google/android/libraries/mobiledatadownload/internal:MddConstants", "//java/com/google/android/libraries/mobiledatadownload/internal:MobileDataDownloadManager", + "//java/com/google/android/libraries/mobiledatadownload/internal/collect", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging/testing:FakeEventLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//java/com/google/android/libraries/mobiledatadownload/internal/util:ProtoConversionUtil", "//java/com/google/android/libraries/mobiledatadownload/lite", "//java/com/google/android/libraries/mobiledatadownload/monitor:DownloadProgressMonitor", + "//java/com/google/common/collect", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:FakeTimeSource", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFlags", "//proto:client_config_java_proto_lite", "//proto:download_config_java_proto_lite", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "@androidx_test", "@com_google_guava_guava", "@com_google_protobuf//:any_proto", @@ -63,7 +72,9 @@ android_local_test( deps = [ "//java/com/google/android/libraries/mobiledatadownload:AggregateException", "//java/com/google/android/libraries/mobiledatadownload:DownloadException", - "@com_google_guava_guava", + "//java/com/google/common/base", + "//java/com/google/common/collect", + "//java/com/google/common/util/concurrent", "@truth", ], ) @@ -77,7 +88,7 @@ android_local_test( }, deps = [ "//java/com/google/android/libraries/mobiledatadownload:DownloadException", - "@com_google_guava_guava", + "//java/com/google/common/util/concurrent", "@truth", ], ) @@ -116,10 +127,13 @@ mdd_android_test( "//java/com/google/android/libraries/mobiledatadownload/tracing", "//javatests/com/google/android/libraries/mobiledatadownload/testing:BlockingFileDownloader", "//javatests/com/google/android/libraries/mobiledatadownload/testing:FakeTimeSource", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:MddTestDependencies", "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFileDownloader", "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFlags", "//proto:client_config_java_proto_lite", "//proto:download_config_java_proto_lite", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "//proto:transform_java_proto_lite", "@android_sdk_linux", "@androidx_core_core", @@ -133,14 +147,58 @@ mdd_android_test( ) mdd_android_test( + name = "MobileDataDownloadIsolatedStructuresIntegrationTest", + size = "large", + srcs = [ + "MobileDataDownloadIsolatedStructuresIntegrationTest.java", + "TestFileGroupPopulator.java", + ], + data = [ + "//javatests/com/google/android/libraries/mobiledatadownload/testdata:integration_test_data_files", + ], + manifest = "//javatests/com/google/android/libraries/mobiledatadownload/testing:AndroidManifest.xml", + target_devices = PARAMETERIZED_EMULATOR_IMAGES, + deps = [ + "//java/com/google/android/libraries/mobiledatadownload", + "//java/com/google/android/libraries/mobiledatadownload:Flags", + "//java/com/google/android/libraries/mobiledatadownload:Logger", + "//java/com/google/android/libraries/mobiledatadownload:MobileDataDownloadBuilder", + "//java/com/google/android/libraries/mobiledatadownload/account:AccountUtil", + "//java/com/google/android/libraries/mobiledatadownload/downloader:FileDownloader", + "//java/com/google/android/libraries/mobiledatadownload/file", + "//java/com/google/android/libraries/mobiledatadownload/file/backends:android", + "//java/com/google/android/libraries/mobiledatadownload/file/backends:file", + "//java/com/google/android/libraries/mobiledatadownload/file/common/testing", + "//java/com/google/android/libraries/mobiledatadownload/monitor:NetworkUsageMonitor", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:FakeTimeSource", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:MddTestDependencies", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFileDownloader", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFlags", + "//proto:client_config_java_proto_lite", + "//proto:download_config_java_proto_lite", + "//third_party/java/testparameterinjector:android", + "@android_sdk_linux", + "@androidx_test", + "@com_google_guava_guava", + "@junit", + "@mockito", + "@truth", + ], +) + +mdd_android_test( name = "DownloadFileGroupIntegrationTest", size = "large", srcs = [ "DownloadFileGroupIntegrationTest.java", "TestFileGroupPopulator.java", ], - manifest = "AndroidManifest.xml", + data = [ + "//javatests/com/google/android/libraries/mobiledatadownload/testdata:downloader_test_data_files", + ], + manifest = "//javatests/com/google/android/libraries/mobiledatadownload/testing:AndroidManifest.xml", tags = ["requires-net:external"], + target_devices = PARAMETERIZED_EMULATOR_IMAGES, deps = [ "//java/com/google/android/libraries/mobiledatadownload", "//java/com/google/android/libraries/mobiledatadownload:AggregateException", @@ -148,23 +206,67 @@ mdd_android_test( "//java/com/google/android/libraries/mobiledatadownload:DownloadListener", "//java/com/google/android/libraries/mobiledatadownload:Flags", "//java/com/google/android/libraries/mobiledatadownload:MobileDataDownloadBuilder", + "//java/com/google/android/libraries/mobiledatadownload/account:AccountUtil", "//java/com/google/android/libraries/mobiledatadownload/downloader:FileDownloader", - "//java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2:base", - "//java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2:base_deps", "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/file/backends:android", - "//java/com/google/android/libraries/mobiledatadownload/file/integration/downloader:downloader2_sp", + "//java/com/google/android/libraries/mobiledatadownload/file/backends:file", "//java/com/google/android/libraries/mobiledatadownload/file/transforms:compress", + "//java/com/google/android/libraries/mobiledatadownload/foreground:ForegroundDownloadKey", "//java/com/google/android/libraries/mobiledatadownload/monitor:DownloadProgressMonitor", "//java/com/google/android/libraries/mobiledatadownload/monitor:NetworkUsageMonitor", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "//javatests/com/google/android/libraries/mobiledatadownload/testing:BlockingFileDownloader", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:MddTestDependencies", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFileDownloader", "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFlags", "//proto:client_config_java_proto_lite", "//proto:download_config_java_proto_lite", + "//proto:transform_java_proto_lite", + "//third_party/java/testparameterinjector:android", + "@android_sdk_linux", + "@androidx_test", + "@com_google_guava_guava", + "@junit", + "@mockito", + "@truth", + ], +) + +mdd_android_test( + name = "DownloadFileGroupCancellationIntegrationTest", + size = "large", + srcs = [ + "DownloadFileGroupCancellationIntegrationTest.java", + "TestFileGroupPopulator.java", + ], + manifest = "//javatests/com/google/android/libraries/mobiledatadownload/testing:AndroidManifest.xml", + target_devices = PARAMETERIZED_EMULATOR_IMAGES, + deps = [ + "//java/com/google/android/libraries/mobiledatadownload", + "//java/com/google/android/libraries/mobiledatadownload:AggregateException", + "//java/com/google/android/libraries/mobiledatadownload:DownloadException", + "//java/com/google/android/libraries/mobiledatadownload:DownloadListener", + "//java/com/google/android/libraries/mobiledatadownload:Flags", + "//java/com/google/android/libraries/mobiledatadownload:MobileDataDownloadBuilder", + "//java/com/google/android/libraries/mobiledatadownload/account:AccountUtil", + "//java/com/google/android/libraries/mobiledatadownload/downloader:FileDownloader", + "//java/com/google/android/libraries/mobiledatadownload/file", + "//java/com/google/android/libraries/mobiledatadownload/file/backends:android", + "//java/com/google/android/libraries/mobiledatadownload/file/transforms:compress", + "//java/com/google/android/libraries/mobiledatadownload/foreground:ForegroundDownloadKey", + "//java/com/google/android/libraries/mobiledatadownload/monitor:DownloadProgressMonitor", + "//java/com/google/android/libraries/mobiledatadownload/monitor:NetworkUsageMonitor", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:BlockingFileDownloader", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:MddTestDependencies", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFlags", + "//proto:client_config_java_proto_lite", + "//proto:download_config_java_proto_lite", + "//third_party/java/testparameterinjector:android", "@android_sdk_linux", "@androidx_test", "@com_google_guava_guava", - "@cronet-api", "@junit", "@mockito", "@truth", @@ -199,10 +301,14 @@ mdd_android_test( "//java/com/google/android/libraries/mobiledatadownload/file/transforms:compress", "//java/com/google/android/libraries/mobiledatadownload/monitor:DownloadProgressMonitor", "//java/com/google/android/libraries/mobiledatadownload/monitor:NetworkUsageMonitor", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:MddTestDependencies", "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFileDownloader", "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFlags", "//proto:client_config_java_proto_lite", "//proto:download_config_java_proto_lite", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", + "//third_party/java/testparameterinjector:android", "@android_sdk_linux", "@androidx_test", "@com_google_guava_guava", @@ -220,10 +326,12 @@ mdd_local_test( "//java/com/google/android/libraries/mobiledatadownload", "//java/com/google/android/libraries/mobiledatadownload:DownloadException", "//java/com/google/android/libraries/mobiledatadownload:DownloadListener", + "//java/com/google/android/libraries/mobiledatadownload:Flags", "//java/com/google/android/libraries/mobiledatadownload:MobileDataDownloadBuilder", "//java/com/google/android/libraries/mobiledatadownload/downloader:FileDownloader", "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/file/backends:android", + "//java/com/google/android/libraries/mobiledatadownload/foreground:ForegroundDownloadKey", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", "//java/com/google/android/libraries/mobiledatadownload/lite:DownloadListener", "//java/com/google/android/libraries/mobiledatadownload/monitor:DownloadProgressMonitor", @@ -245,8 +353,9 @@ mdd_android_test( srcs = [ "DownloadFileIntegrationTest.java", ], - manifest = "AndroidManifest.xml", + manifest = "//javatests/com/google/android/libraries/mobiledatadownload/testing:AndroidManifest.xml", tags = ["requires-net:external"], + target_devices = PARAMETERIZED_EMULATOR_IMAGES, deps = [ "//java/com/google/android/libraries/mobiledatadownload", "//java/com/google/android/libraries/mobiledatadownload:AggregateException", @@ -261,13 +370,16 @@ mdd_android_test( "//java/com/google/android/libraries/mobiledatadownload/file/backends:android", "//java/com/google/android/libraries/mobiledatadownload/file/integration/downloader:downloader2_sp", "//java/com/google/android/libraries/mobiledatadownload/file/transforms:compress", + "//java/com/google/android/libraries/mobiledatadownload/foreground:ForegroundDownloadKey", "//java/com/google/android/libraries/mobiledatadownload/monitor:DownloadProgressMonitor", "//java/com/google/android/libraries/mobiledatadownload/monitor:NetworkUsageMonitor", "//javatests/com/google/android/libraries/mobiledatadownload/testing:BlockingFileDownloader", "//javatests/com/google/android/libraries/mobiledatadownload/testing:FakeTimeSource", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:MddTestDependencies", "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFlags", "//proto:client_config_java_proto_lite", "//proto:download_config_java_proto_lite", + "//third_party/java/testparameterinjector:android", "@android_sdk_linux", "@androidx_test", "@com_google_guava_guava", @@ -289,10 +401,12 @@ mdd_android_test( "//javatests/com/google/android/libraries/mobiledatadownload/testdata:integration_test_data_files", ], manifest = "//javatests/com/google/android/libraries/mobiledatadownload/testing:AndroidManifest.xml", + target_devices = PARAMETERIZED_EMULATOR_IMAGES, deps = [ "//java/com/google/android/libraries/mobiledatadownload", "//java/com/google/android/libraries/mobiledatadownload:AggregateException", "//java/com/google/android/libraries/mobiledatadownload:DownloadException", + "//java/com/google/android/libraries/mobiledatadownload:ExperimentationConfig", "//java/com/google/android/libraries/mobiledatadownload:FileSource", "//java/com/google/android/libraries/mobiledatadownload:Flags", "//java/com/google/android/libraries/mobiledatadownload:MobileDataDownloadBuilder", @@ -310,14 +424,17 @@ mdd_android_test( "//java/com/google/android/libraries/mobiledatadownload/monitor:DownloadProgressMonitor", "//java/com/google/android/libraries/mobiledatadownload/monitor:NetworkUsageMonitor", "//javatests/com/google/android/libraries/mobiledatadownload/testing:BlockingFileDownloader", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:MddTestDependencies", "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFlags", "//proto:client_config_java_proto_lite", "//proto:download_config_java_proto_lite", + "//third_party/java/testparameterinjector:android", "@android_sdk_linux", "@androidx_test", "@com_google_guava_guava", "@com_google_protobuf//:protobuf_lite", "@cronet-api", + "@javax_inject", "@junit", "@mockito", "@truth", @@ -354,10 +471,14 @@ mdd_android_test( "//java/com/google/android/libraries/mobiledatadownload/monitor:DownloadProgressMonitor", "//java/com/google/android/libraries/mobiledatadownload/monitor:NetworkUsageMonitor", "//javatests/com/google/android/libraries/mobiledatadownload/internal:MddTestUtil", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:MddTestDependencies", "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFileDownloader", "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFlags", "//proto:client_config_java_proto_lite", "//proto:download_config_java_proto_lite", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", + "//third_party/java/testparameterinjector:android", "@android_sdk_linux", "@androidx_test", "@com_google_guava_guava", diff --git a/javatests/com/google/android/libraries/mobiledatadownload/DownloadFileGroupAndroidSharingIntegrationTest.java b/javatests/com/google/android/libraries/mobiledatadownload/DownloadFileGroupAndroidSharingIntegrationTest.java index 503d573..7e053ad 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/DownloadFileGroupAndroidSharingIntegrationTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/DownloadFileGroupAndroidSharingIntegrationTest.java @@ -19,15 +19,18 @@ import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopul import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_ID; import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_SIZE; import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_URL; +import static com.google.android.libraries.mobiledatadownload.testing.MddTestDependencies.ExecutorType; import static com.google.common.truth.Truth.assertThat; import static java.util.concurrent.TimeUnit.SECONDS; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import android.app.blob.BlobStoreManager; import android.content.Context; import android.net.Uri; import android.util.Log; import androidx.test.core.app.ApplicationProvider; -import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.android.libraries.mobiledatadownload.downloader.FileDownloader; import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; import com.google.android.libraries.mobiledatadownload.file.backends.AndroidFileBackend; @@ -49,9 +52,13 @@ import com.google.mobiledatadownload.ClientConfigProto.ClientFile; import com.google.mobiledatadownload.ClientConfigProto.ClientFileGroup; import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; import com.google.mobiledatadownload.DownloadConfigProto.DownloadConditions.DeviceNetworkPolicy; +import com.google.mobiledatadownload.LogProto.MddLogData; +import com.google.testing.junit.testparameterinjector.TestParameter; +import com.google.testing.junit.testparameterinjector.TestParameterInjector; import java.io.OutputStream; import java.security.MessageDigest; import java.util.Calendar; +import java.util.List; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import org.junit.After; @@ -59,11 +66,12 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; -@RunWith(AndroidJUnit4.class) +@RunWith(TestParameterInjector.class) public class DownloadFileGroupAndroidSharingIntegrationTest { private static final String TAG = "DownloadFileGroupIntegrationTest"; @@ -72,9 +80,6 @@ public class DownloadFileGroupAndroidSharingIntegrationTest { private static final String TEST_DATA_RELATIVE_PATH = "third_party/java_src/android_libs/mobiledatadownload/javatests/com/google/android/libraries/mobiledatadownload/testdata/"; - // Note: Control Executor must not be a single thread executor. - private static final ListeningExecutorService CONTROL_EXECUTOR = - MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); private static final ScheduledExecutorService DOWNLOAD_EXECUTOR = Executors.newScheduledThreadPool(2); @@ -106,15 +111,24 @@ public class DownloadFileGroupAndroidSharingIntegrationTest { private BlobStoreBackend blobStoreBackend; private BlobStoreManager blobStoreManager; private MobileDataDownload mobileDataDownload; + private ListeningExecutorService controlExecutor; private final TestFlags flags = new TestFlags(); @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + // TODO(b/226405643): Some tests seem to fail due to BlobStore not clearing out files across runs. + // Investigate why this is happening and enable single-threaded tests. + @TestParameter({"MULTI_THREADED"}) + ExecutorType controlExecutorType; + @Before public void setUp() throws Exception { + flags.mddAndroidSharingSampleInterval = Optional.of(1); + flags.mddDefaultSampleInterval = Optional.of(1); + blobStoreBackend = new BlobStoreBackend(context); blobStoreManager = (BlobStoreManager) context.getSystemService(Context.BLOB_STORE_SERVICE); @@ -127,26 +141,7 @@ public class DownloadFileGroupAndroidSharingIntegrationTest { /* transforms= */ ImmutableList.of(new CompressTransform()), /* monitors= */ ImmutableList.of(mockNetworkUsageMonitor, mockDownloadProgressMonitor)); - Supplier<FileDownloader> fileDownloaderSupplier = - () -> - new TestFileDownloader( - TEST_DATA_RELATIVE_PATH, - fileStorage, - MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR)); - - mobileDataDownload = - MobileDataDownloadBuilder.newBuilder() - .setContext(context) - .setControlExecutor(CONTROL_EXECUTOR) - .setFileDownloaderSupplier(fileDownloaderSupplier) - .setTaskScheduler(Optional.of(mockTaskScheduler)) - .setDeltaDecoderOptional(Optional.absent()) - .setFileStorage(fileStorage) - .setNetworkUsageMonitor(mockNetworkUsageMonitor) - .setDownloadMonitorOptional(Optional.of(mockDownloadProgressMonitor)) - .setLoggerOptional(Optional.of(mockLogger)) - .setFlagsOptional(Optional.of(flags)) - .build(); + controlExecutor = controlExecutorType.executor(); } @After @@ -172,26 +167,7 @@ public class DownloadFileGroupAndroidSharingIntegrationTest { /* transforms= */ ImmutableList.of(new CompressTransform()), /* monitors= */ ImmutableList.of(mockNetworkUsageMonitor, mockDownloadProgressMonitor)); - Supplier<FileDownloader> fileDownloaderSupplier = - () -> - new TestFileDownloader( - TEST_DATA_RELATIVE_PATH, - fileStorage, - MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR)); - - mobileDataDownload = - MobileDataDownloadBuilder.newBuilder() - .setContext(context) - .setControlExecutor(CONTROL_EXECUTOR) - .setFileDownloaderSupplier(fileDownloaderSupplier) - .setTaskScheduler(Optional.of(mockTaskScheduler)) - .setDeltaDecoderOptional(Optional.absent()) - .setFileStorage(fileStorage) - .setNetworkUsageMonitor(mockNetworkUsageMonitor) - .setDownloadMonitorOptional(Optional.of(mockDownloadProgressMonitor)) - .setLoggerOptional(Optional.of(mockLogger)) - .setFlagsOptional(Optional.of(flags)) - .build(); + mobileDataDownload = builderForTest().setFileStorage(fileStorage).build(); Uri androidUri = BlobUri.builder(context).setBlobParameters(FILE_ANDROID_SHARING_CHECKSUM_1).build(); @@ -255,10 +231,25 @@ public class DownloadFileGroupAndroidSharingIntegrationTest { assertThat(clientFile.getFileId()).isEqualTo(FILE_ID); uri = Uri.parse(clientFile.getFileUri()); assertThat(fileStorage.fileSize(uri)).isEqualTo(FILE_SIZE); + + ArgumentCaptor<MddLogData> logDataCaptor = ArgumentCaptor.forClass(MddLogData.class); + // 1073 is the tag number for MddClientEvent.Code.EVENT_CODE_UNSPECIFIED. + verify(mockLogger, times(2)).log(logDataCaptor.capture(), /* eventCode= */ eq(1073)); + + List<MddLogData> logData = logDataCaptor.getAllValues(); + Void log1 = null; + Void log2 = null; + assertThat(logData).hasSize(2); + + Void androidSharingLog = null; + assertThat(log1).isEqualTo(androidSharingLog); + assertThat(log2).isEqualTo(androidSharingLog); } @Test public void oneAndroidSharedFile_twoFileGroups_downloadedOnlyOnce() throws Exception { + mobileDataDownload = builderForTest().build(); + Uri androidUri = BlobUri.builder(context).setBlobParameters(FILE_ANDROID_SHARING_CHECKSUM_1).build(); assertThat(fileStorage.exists(androidUri)).isFalse(); @@ -398,10 +389,22 @@ public class DownloadFileGroupAndroidSharingIntegrationTest { assertThat(uri).isEqualTo(androidUri); assertThat(fileStorage.exists(uri)).isTrue(); assertThat(blobStoreManager.getLeasedBlobs()).hasSize(1); + + ArgumentCaptor<MddLogData> logDataCaptor = ArgumentCaptor.forClass(MddLogData.class); + // 1073 is the tag number for MddClientEvent.Code.EVENT_CODE_UNSPECIFIED. + verify(mockLogger, times(2)).log(logDataCaptor.capture(), /* eventCode= */ eq(1073)); + + List<MddLogData> logData = logDataCaptor.getAllValues(); + assertThat(logData).hasSize(2); + + Void log1 = null; + Void log2 = null; } @Test public void fileAvailableInSharedStorage_neverDownloaded() throws Exception { + mobileDataDownload = builderForTest().build(); + byte[] content = "fileAvailableInSharedStorage_neverDownloaded".getBytes(); String androidChecksum = computeDigest(content, "SHA-256"); String checksum = computeDigest(content, "SHA-1"); @@ -481,10 +484,21 @@ public class DownloadFileGroupAndroidSharingIntegrationTest { assertThat(clientFile.getFileId()).isEqualTo(FILE_ID); uri = Uri.parse(clientFile.getFileUri()); assertThat(fileStorage.fileSize(uri)).isEqualTo(FILE_SIZE); + + ArgumentCaptor<MddLogData> logDataCaptor = ArgumentCaptor.forClass(MddLogData.class); + // 1073 is the tag number for MddClientEvent.Code.EVENT_CODE_UNSPECIFIED. + verify(mockLogger).log(logDataCaptor.capture(), /* eventCode= */ eq(1073)); + + List<MddLogData> logData = logDataCaptor.getAllValues(); + assertThat(logData).hasSize(1); + + Void log1 = null; } @Test public void fileDownloadedForFirstFileGroup_thenSharedForSecondFileGroup() throws Exception { + mobileDataDownload = builderForTest().build(); + Uri androidUri = BlobUri.builder(context).setBlobParameters(FILE_ANDROID_SHARING_CHECKSUM_2).build(); assertThat(blobStoreBackend.exists(androidUri)).isFalse(); @@ -618,5 +632,35 @@ public class DownloadFileGroupAndroidSharingIntegrationTest { assertThat(uri).isEqualTo(androidUri); assertThat(fileStorage.exists(uri)).isTrue(); assertThat(blobStoreManager.getLeasedBlobs()).hasSize(1); + + ArgumentCaptor<MddLogData> logDataCaptor = ArgumentCaptor.forClass(MddLogData.class); + // 1073 is the tag number for MddClientEvent.Code.EVENT_CODE_UNSPECIFIED. + verify(mockLogger).log(logDataCaptor.capture(), /* eventCode= */ eq(1073)); + + List<MddLogData> logData = logDataCaptor.getAllValues(); + assertThat(logData).hasSize(1); + + Void log1 = null; + } + + private MobileDataDownloadBuilder builderForTest() { + Supplier<FileDownloader> fileDownloaderSupplier = + () -> + new TestFileDownloader( + TEST_DATA_RELATIVE_PATH, + fileStorage, + MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR)); + + return MobileDataDownloadBuilder.newBuilder() + .setContext(context) + .setControlExecutor(controlExecutor) + .setFileDownloaderSupplier(fileDownloaderSupplier) + .setTaskScheduler(Optional.of(mockTaskScheduler)) + .setDeltaDecoderOptional(Optional.absent()) + .setFileStorage(fileStorage) + .setNetworkUsageMonitor(mockNetworkUsageMonitor) + .setDownloadMonitorOptional(Optional.of(mockDownloadProgressMonitor)) + .setLoggerOptional(Optional.of(mockLogger)) + .setFlagsOptional(Optional.of(flags)); } } diff --git a/javatests/com/google/android/libraries/mobiledatadownload/DownloadFileGroupCancellationIntegrationTest.java b/javatests/com/google/android/libraries/mobiledatadownload/DownloadFileGroupCancellationIntegrationTest.java new file mode 100644 index 0000000..96e532e --- /dev/null +++ b/javatests/com/google/android/libraries/mobiledatadownload/DownloadFileGroupCancellationIntegrationTest.java @@ -0,0 +1,209 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload; + +import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_GROUP_NAME; +import static com.google.android.libraries.mobiledatadownload.testing.MddTestDependencies.ExecutorType; +import static com.google.common.truth.Truth.assertThat; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.junit.Assert.fail; + +import android.accounts.Account; +import android.content.Context; +import android.util.Log; +import androidx.test.core.app.ApplicationProvider; +import com.google.android.libraries.mobiledatadownload.account.AccountUtil; +import com.google.android.libraries.mobiledatadownload.downloader.DownloadRequest; +import com.google.android.libraries.mobiledatadownload.downloader.FileDownloader; +import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; +import com.google.android.libraries.mobiledatadownload.file.backends.AndroidFileBackend; +import com.google.android.libraries.mobiledatadownload.file.transforms.CompressTransform; +import com.google.android.libraries.mobiledatadownload.monitor.DownloadProgressMonitor; +import com.google.android.libraries.mobiledatadownload.monitor.NetworkUsageMonitor; +import com.google.android.libraries.mobiledatadownload.testing.BlockingFileDownloader; +import com.google.android.libraries.mobiledatadownload.testing.TestFlags; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.ListeningScheduledExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.mobiledatadownload.ClientConfigProto.ClientFileGroup; +import com.google.testing.junit.testparameterinjector.TestParameter; +import com.google.testing.junit.testparameterinjector.TestParameterInjector; +import java.util.concurrent.Executors; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** + * Integration Tests that relate to download cancellation should be placed here. + * + * <p>This includes calling {@link MobileDataDownload#cancelForegroundDownload} for cancelling the + * future returned from {@link MobileDataDownload#downloadFileGroup} or {@link + * MobileDataDownload#downloadFileGroupWithForegroundService}. + */ +@RunWith(TestParameterInjector.class) +public class DownloadFileGroupCancellationIntegrationTest { + + private static final String TAG = "DownloadFileGroupCancellationIntegrationTest"; + private static final int MAX_DOWNLOAD_FILE_GROUP_WAIT_TIME_SECS = 60; + private static final long MAX_MDD_API_WAIT_TIME_SECS = 5L; + + private static final ListeningScheduledExecutorService DOWNLOAD_EXECUTOR = + MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(4)); + + private static final String FILE_GROUP_NAME_INSECURE_URL = "test-group-insecure-url"; + private static final String FILE_GROUP_NAME_MULTIPLE_FILES = "test-group-multiple-files"; + + private static final String FILE_ID_1 = "test-file-1"; + private static final String FILE_ID_2 = "test-file-2"; + private static final String FILE_CHECKSUM_1 = "a1cba9d87b1440f41ce9e7da38c43e1f6bd7d5df"; + private static final String FILE_CHECKSUM_2 = "cb2459d9f1b508993aba36a5ffd942a7e0d49ed6"; + private static final String FILE_NOT_EXIST_URL = + "https://www.gstatic.com/icing/idd/notexist/file.txt"; + + private static final String VARIANT_1 = "test-variant-1"; + private static final String VARIANT_2 = "test-variant-2"; + + private static final Account ACCOUNT_1 = AccountUtil.create("account-name-1", "account-type"); + private static final Account ACCOUNT_2 = AccountUtil.create("account-name-2", "account-type"); + + private static final Context context = ApplicationProvider.getApplicationContext(); + + @Mock private TaskScheduler mockTaskScheduler; + @Mock private NetworkUsageMonitor mockNetworkUsageMonitor; + @Mock private DownloadProgressMonitor mockDownloadProgressMonitor; + + private SynchronousFileStorage fileStorage; + private ListeningExecutorService controlExecutor; + + private final TestFlags flags = new TestFlags(); + + @Rule(order = 1) + public final MockitoRule mocks = MockitoJUnit.rule(); + + @TestParameter ExecutorType controlExecutorType; + + @Before + public void setUp() throws Exception { + + fileStorage = + new SynchronousFileStorage( + /* backends= */ ImmutableList.of(AndroidFileBackend.builder(context).build()), + /* transforms= */ ImmutableList.of(new CompressTransform()), + /* monitors= */ ImmutableList.of(mockNetworkUsageMonitor, mockDownloadProgressMonitor)); + + controlExecutor = controlExecutorType.executor(); + } + + @Test + public void cancelDownload() throws Exception { + // In this test we will start a download and make sure that calling cancel on the returned + // future will cancel the download. + // We create a BlockingFileDownloader that allows the download to be blocked indefinitely. + // We also provide a delegate FileDownloader that attaches a FutureCallback to the internal + // download future and fail if the future is not cancelled. + BlockingFileDownloader blockingFileDownloader = + new BlockingFileDownloader( + DOWNLOAD_EXECUTOR, + new FileDownloader() { + @Override + public ListenableFuture<Void> startDownloading(DownloadRequest downloadRequest) { + ListenableFuture<Void> downloadTaskFuture = Futures.immediateVoidFuture(); + PropagatedFutures.addCallback( + downloadTaskFuture, + new FutureCallback<Void>() { + @Override + public void onSuccess(Void result) { + // Should not get here since we will cancel the future. + fail(); + } + + @Override + public void onFailure(Throwable t) { + assertThat(downloadTaskFuture.isCancelled()).isTrue(); + + Log.i(TAG, "downloadTask is cancelled!"); + } + }, + DOWNLOAD_EXECUTOR); + return downloadTaskFuture; + } + }); + + // Use never finish downloader to test whether the cancellation on the downloadFuture would + // cancel all the parent futures. + TestFileGroupPopulator testFileGroupPopulator = new TestFileGroupPopulator(context); + MobileDataDownload mobileDataDownload = + builderForTest() + .setFileDownloaderSupplier(() -> blockingFileDownloader) + .addFileGroupPopulator(testFileGroupPopulator) + .build(); + + testFileGroupPopulator + .refreshFileGroups(mobileDataDownload) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); + + // Now start to download the file group. + ListenableFuture<ClientFileGroup> downloadFileGroupFuture = + mobileDataDownload.downloadFileGroup( + DownloadFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()); + + // Note: we could have a race condition here between when we call the + // downloadFileGroupFuture.cancel and when the FileDownloader.startDownloading is executed. + // The following call will ensure that we will only call cancel on the downloadFileGroupFuture + // when the actual download has happened (the downloadTaskFuture). + // This will block until the downloadTaskFuture starts. + blockingFileDownloader.waitForDownloadStarted(); + + // Cancel the downloadFileGroupFuture, it should cascade cancellation to downloadTaskFuture. + downloadFileGroupFuture.cancel(true /*may interrupt*/); + + // Allow the download to continue and trigger our delegate FileDownloader. If the future isn't + // cancelled, the onSuccess callback should fail the test. + blockingFileDownloader.finishDownloading(); + blockingFileDownloader.waitForDownloadCompleted(); + + assertThat(downloadFileGroupFuture.isCancelled()).isTrue(); + + mobileDataDownload.clear().get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); + } + + /** + * Returns MDD Builder with common dependencies set -- additional dependencies are added in each + * test as needed. + */ + private MobileDataDownloadBuilder builderForTest() { + + return MobileDataDownloadBuilder.newBuilder() + .setContext(context) + .setControlExecutor(controlExecutor) + .setFileStorage(fileStorage) + .setTaskScheduler(Optional.of(mockTaskScheduler)) + .setDeltaDecoderOptional(Optional.absent()) + .setNetworkUsageMonitor(mockNetworkUsageMonitor) + .setDownloadMonitorOptional(Optional.of(mockDownloadProgressMonitor)) + .setFlagsOptional(Optional.of(flags)); + } +} diff --git a/javatests/com/google/android/libraries/mobiledatadownload/DownloadFileGroupIntegrationTest.java b/javatests/com/google/android/libraries/mobiledatadownload/DownloadFileGroupIntegrationTest.java index c5b3239..818b2e4 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/DownloadFileGroupIntegrationTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/DownloadFileGroupIntegrationTest.java @@ -20,42 +20,51 @@ import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopul import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_ID; import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_SIZE; import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_URL; +import static com.google.android.libraries.mobiledatadownload.testing.MddTestDependencies.DownloaderConfigurationType; +import static com.google.android.libraries.mobiledatadownload.testing.MddTestDependencies.ExecutorType; import static com.google.common.truth.Truth.assertThat; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.Assert.assertThrows; -import static org.junit.Assert.fail; +import android.accounts.Account; import android.content.Context; import android.net.Uri; import android.util.Log; import androidx.test.core.app.ApplicationProvider; -import androidx.test.ext.junit.runners.AndroidJUnit4; -import com.google.android.libraries.mobiledatadownload.downloader.DownloadRequest; +import com.google.android.libraries.mobiledatadownload.account.AccountUtil; import com.google.android.libraries.mobiledatadownload.downloader.FileDownloader; -import com.google.android.libraries.mobiledatadownload.downloader.offroad.dagger.downloader2.BaseFileDownloaderModule; import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; import com.google.android.libraries.mobiledatadownload.file.backends.AndroidFileBackend; -import com.google.android.libraries.mobiledatadownload.file.integration.downloader.SharedPreferencesDownloadMetadata; +import com.google.android.libraries.mobiledatadownload.file.backends.JavaFileBackend; import com.google.android.libraries.mobiledatadownload.file.transforms.CompressTransform; import com.google.android.libraries.mobiledatadownload.monitor.DownloadProgressMonitor; import com.google.android.libraries.mobiledatadownload.monitor.NetworkUsageMonitor; import com.google.android.libraries.mobiledatadownload.testing.BlockingFileDownloader; +import com.google.android.libraries.mobiledatadownload.testing.MddTestDependencies; +import com.google.android.libraries.mobiledatadownload.testing.TestFileDownloader; import com.google.android.libraries.mobiledatadownload.testing.TestFlags; import com.google.common.base.Optional; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.mobiledatadownload.ClientConfigProto.ClientFile; import com.google.mobiledatadownload.ClientConfigProto.ClientFileGroup; +import com.google.mobiledatadownload.DownloadConfigProto.DataFile; import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; import com.google.mobiledatadownload.DownloadConfigProto.DownloadConditions.DeviceNetworkPolicy; +import com.google.mobiledatadownload.TransformProto; +import com.google.mobiledatadownload.TransformProto.Transform; +import com.google.mobiledatadownload.TransformProto.Transforms; +import com.google.testing.junit.testparameterinjector.TestParameter; +import com.google.testing.junit.testparameterinjector.TestParameterInjector; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -64,19 +73,22 @@ import org.mockito.Mock; import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; -@RunWith(AndroidJUnit4.class) +/** + * Integration Tests that relate to {@link MobileDataDownload#downloadFileGroup}. + * + * <p>NOTE: Any tests related to cancellation should be added to {@link + * DownloadFileGroupCancellationIntegrationTest} instead. + */ +@RunWith(TestParameterInjector.class) public class DownloadFileGroupIntegrationTest { private static final String TAG = "DownloadFileGroupIntegrationTest"; - private static final int MAX_DOWNLOAD_FILE_GROUP_WAIT_TIME_SECS = 300; + private static final int MAX_DOWNLOAD_FILE_GROUP_WAIT_TIME_SECS = 60; + private static final int MAX_MULTI_MDD_API_WAIT_TIME_SECS = 120; + private static final long MAX_MDD_API_WAIT_TIME_SECS = 5L; - // Note: Control Executor must not be a single thread executor. - private static final ListeningExecutorService CONTROL_EXECUTOR = - MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); - private static final ScheduledExecutorService DOWNLOAD_EXECUTOR = - Executors.newScheduledThreadPool(2); - private static final ListeningExecutorService listeningExecutorService = - MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR); + private static final ListeningScheduledExecutorService DOWNLOAD_EXECUTOR = + MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(4)); private static final String FILE_GROUP_NAME_INSECURE_URL = "test-group-insecure-url"; private static final String FILE_GROUP_NAME_MULTIPLE_FILES = "test-group-multiple-files"; @@ -88,6 +100,24 @@ public class DownloadFileGroupIntegrationTest { private static final String FILE_NOT_EXIST_URL = "https://www.gstatic.com/icing/idd/notexist/file.txt"; + private static final String TEST_DATA_RELATIVE_PATH = + "third_party/java_src/android_libs/mobiledatadownload/javatests/com/google/android/libraries/mobiledatadownload/testdata/"; + + private static final String TEST_DATA_URL = "https://test.url/full_file.txt"; + private static final String TEST_DATA_CHECKSUM = "0c4f1e55c4ec28d0305c5cfde8610b7e6e9f7d9a"; + private static final int TEST_DATA_BYTE_SIZE = 110; + + private static final String TEST_DATA_COMPRESS_URL = "https://test.url/full_file.zlib"; + private static final String TEST_DATA_COMPRESS_CHECKSUM = + "cbffcf480fd52a3c6bf9d21206d36f0a714bb97a"; + private static final int TEST_DATA_COMPRESS_BYTE_SIZE = 92; + + private static final String VARIANT_1 = "test-variant-1"; + private static final String VARIANT_2 = "test-variant-2"; + + private static final Account ACCOUNT_1 = AccountUtil.create("account-name-1", "account-type"); + private static final Account ACCOUNT_2 = AccountUtil.create("account-name-2", "account-type"); + private static final Context context = ApplicationProvider.getApplicationContext(); @Mock private TaskScheduler mockTaskScheduler; @@ -95,81 +125,111 @@ public class DownloadFileGroupIntegrationTest { @Mock private DownloadProgressMonitor mockDownloadProgressMonitor; private SynchronousFileStorage fileStorage; + private ListeningExecutorService controlExecutor; private final TestFlags flags = new TestFlags(); - @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + @Rule(order = 1) + public final MockitoRule mocks = MockitoJUnit.rule(); - /* Differentiates between Downloader libraries for shared test method assertions. */ - private enum DownloaderVersion { - V2 - } + @TestParameter ExecutorType controlExecutorType; @Before public void setUp() throws Exception { fileStorage = new SynchronousFileStorage( - /* backends= */ ImmutableList.of(AndroidFileBackend.builder(context).build()), + /* backends= */ ImmutableList.of( + AndroidFileBackend.builder(context).build(), new JavaFileBackend()), /* transforms= */ ImmutableList.of(new CompressTransform()), /* monitors= */ ImmutableList.of(mockNetworkUsageMonitor, mockDownloadProgressMonitor)); - } - @Test - public void downloadAndRead_downloader2() throws Exception { - Supplier<FileDownloader> fileDownloaderSupplier = - () -> - BaseFileDownloaderModule.createOffroad2FileDownloader( - context, - DOWNLOAD_EXECUTOR, - CONTROL_EXECUTOR, - fileStorage, - new SharedPreferencesDownloadMetadata( - context.getSharedPreferences("downloadmetadata", 0), listeningExecutorService), - Optional.of(mockDownloadProgressMonitor), - /* urlEngineOptional= */ Optional.absent(), - /* exceptionHandlerOptional= */ Optional.absent(), - /* authTokenProviderOptional= */ Optional.absent(), - /* trafficTag= */ Optional.absent(), - flags); - - testDownloadAndRead(fileDownloaderSupplier, DownloaderVersion.V2); + controlExecutor = controlExecutorType.executor(); } @Test - public void downloadFailed_downloader2() throws Exception { - Supplier<FileDownloader> fileDownloaderSupplier = - () -> - BaseFileDownloaderModule.createOffroad2FileDownloader( - context, - DOWNLOAD_EXECUTOR, - CONTROL_EXECUTOR, - fileStorage, - new SharedPreferencesDownloadMetadata( - context.getSharedPreferences("downloadmetadata", 0), listeningExecutorService), - Optional.of(mockDownloadProgressMonitor), - /* urlEngineOptional= */ Optional.absent(), - /* exceptionHandlerOptional= */ Optional.absent(), - /* authTokenProviderOptional= */ Optional.absent(), - /* trafficTag= */ Optional.absent(), - flags); - - testDownloadFailed(fileDownloaderSupplier, DownloaderVersion.V2); + public void downloadAndRead( + @TestParameter DownloaderConfigurationType downloaderConfigurationType) throws Exception { + Optional<String> instanceId = Optional.of(MddTestDependencies.randomInstanceId()); + TestFileGroupPopulator testFileGroupPopulator = new TestFileGroupPopulator(context); + MobileDataDownload mobileDataDownload = + builderForTest() + .setInstanceIdOptional(instanceId) + .setFileDownloaderSupplier( + downloaderConfigurationType.fileDownloaderSupplier( + context, + controlExecutor, + DOWNLOAD_EXECUTOR, + fileStorage, + flags, + Optional.of(mockDownloadProgressMonitor), + instanceId)) + .addFileGroupPopulator(testFileGroupPopulator) + .build(); + + testFileGroupPopulator + .refreshFileGroups(mobileDataDownload) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); + + mobileDataDownload + .downloadFileGroup( + DownloadFileGroupRequest.newBuilder() + .setGroupName(FILE_GROUP_NAME) + .setListenerOptional( + Optional.of( + new DownloadListener() { + @Override + public void onProgress(long currentSize) { + Log.i(TAG, "onProgress " + currentSize); + } + + @Override + public void onComplete(ClientFileGroup clientFileGroup) { + Log.i(TAG, "onComplete " + clientFileGroup.getGroupName()); + } + })) + .build()) + .get(MAX_DOWNLOAD_FILE_GROUP_WAIT_TIME_SECS, SECONDS); + + ClientFileGroup clientFileGroup = + mobileDataDownload + .getFileGroup(GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); + + assertThat(clientFileGroup).isNotNull(); + assertThat(clientFileGroup.getGroupName()).isEqualTo(FILE_GROUP_NAME); + assertThat(clientFileGroup.getFileCount()).isEqualTo(1); + + ClientFile clientFile = clientFileGroup.getFileList().get(0); + assertThat(clientFile.getFileId()).isEqualTo(FILE_ID); + Uri androidUri = Uri.parse(clientFile.getFileUri()); + assertThat(fileStorage.fileSize(androidUri)).isEqualTo(FILE_SIZE); + + mobileDataDownload.clear().get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); + + switch (downloaderConfigurationType) { + case V2_PLATFORM: + // No-op + } } - private void testDownloadFailed( - Supplier<FileDownloader> fileDownloaderSupplier, DownloaderVersion version) throws Exception { + @Test + public void downloadFailed() throws Exception { + // NOTE: The test failures here are not network stack dependent, so there's + // no need to parameterize this test for different network stacks. + Optional<String> instanceId = Optional.of(MddTestDependencies.randomInstanceId()); MobileDataDownload mobileDataDownload = - MobileDataDownloadBuilder.newBuilder() - .setContext(context) - .setControlExecutor(CONTROL_EXECUTOR) - .setFileDownloaderSupplier(fileDownloaderSupplier) - .setTaskScheduler(Optional.of(mockTaskScheduler)) - .setDeltaDecoderOptional(Optional.absent()) - .setFileStorage(fileStorage) - .setNetworkUsageMonitor(mockNetworkUsageMonitor) - .setDownloadMonitorOptional(Optional.of(mockDownloadProgressMonitor)) - .setFlagsOptional(Optional.of(flags)) + builderForTest() + .setInstanceIdOptional(instanceId) + .setFileDownloaderSupplier( + DownloaderConfigurationType.V2_PLATFORM.fileDownloaderSupplier( + context, + controlExecutor, + DOWNLOAD_EXECUTOR, + fileStorage, + flags, + Optional.of(mockDownloadProgressMonitor), + instanceId)) .build(); // The data file group has a file with insecure url. @@ -200,7 +260,7 @@ public class DownloadFileGroupIntegrationTest { mobileDataDownload .addFileGroup( AddFileGroupRequest.newBuilder().setDataFileGroup(groupWithInsecureUrl).build()) - .get()) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS)) .isTrue(); assertThat( @@ -209,7 +269,7 @@ public class DownloadFileGroupIntegrationTest { AddFileGroupRequest.newBuilder() .setDataFileGroup(groupWithMultipleFiles) .build()) - .get()) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS)) .isTrue(); ExecutionException exception = @@ -221,7 +281,7 @@ public class DownloadFileGroupIntegrationTest { DownloadFileGroupRequest.newBuilder() .setGroupName(FILE_GROUP_NAME_INSECURE_URL) .build()) - .get(MAX_DOWNLOAD_FILE_GROUP_WAIT_TIME_SECS, TimeUnit.SECONDS)); + .get(MAX_DOWNLOAD_FILE_GROUP_WAIT_TIME_SECS, SECONDS)); assertThat(exception).hasCauseThat().isInstanceOf(AggregateException.class); AggregateException cause = (AggregateException) exception.getCause(); assertThat(cause).isNotNull(); @@ -239,173 +299,264 @@ public class DownloadFileGroupIntegrationTest { DownloadFileGroupRequest.newBuilder() .setGroupName(FILE_GROUP_NAME_MULTIPLE_FILES) .build()) - .get(MAX_DOWNLOAD_FILE_GROUP_WAIT_TIME_SECS, TimeUnit.SECONDS)); + .get(MAX_DOWNLOAD_FILE_GROUP_WAIT_TIME_SECS, SECONDS)); assertThat(exception2).hasCauseThat().isInstanceOf(AggregateException.class); AggregateException cause2 = (AggregateException) exception2.getCause(); assertThat(cause2).isNotNull(); ImmutableList<Throwable> failures2 = cause2.getFailures(); assertThat(failures2).hasSize(2); assertThat(failures2.get(0)).isInstanceOf(DownloadException.class); - switch (version) { - case V2: - assertThat(failures2.get(0)) - .hasCauseThat() - .hasMessageThat() - .containsMatch("httpStatusCode=404"); - break; - } + assertThat(failures2.get(0)) + .hasCauseThat() + .hasMessageThat() + .containsMatch("httpStatusCode=404"); assertThat(failures2.get(1)).isInstanceOf(DownloadException.class); assertThat(failures2.get(1)).hasMessageThat().contains("INSECURE_URL_ERROR"); - switch (version) { - case V2: - // No-op - } + AggregateException exception3 = + assertThrows( + AggregateException.class, + () -> { + try { + ListenableFuture<ClientFileGroup> downloadFuture1 = + mobileDataDownload.downloadFileGroup( + DownloadFileGroupRequest.newBuilder() + .setGroupName(FILE_GROUP_NAME_MULTIPLE_FILES) + .build()); + ListenableFuture<ClientFileGroup> downloadFuture2 = + mobileDataDownload.downloadFileGroup( + DownloadFileGroupRequest.newBuilder() + .setGroupName(FILE_GROUP_NAME_INSECURE_URL) + .build()); + + Futures.successfulAsList(downloadFuture1, downloadFuture2) + .get(MAX_DOWNLOAD_FILE_GROUP_WAIT_TIME_SECS, SECONDS); + + AggregateException.throwIfFailed( + ImmutableList.of(downloadFuture1, downloadFuture2), + "Expected download failures"); + } catch (ExecutionException e) { + throw e; + } + }); + assertThat(exception3.getFailures()).hasSize(2); } - private void testDownloadAndRead( - Supplier<FileDownloader> fileDownloaderSupplier, DownloaderVersion version) throws Exception { - TestFileGroupPopulator testFileGroupPopulator = new TestFileGroupPopulator(context); + @Test + public void removePartialDownloadThenDownloadAgain( + @TestParameter DownloaderConfigurationType downloaderConfigurationType) throws Exception { + Optional<String> instanceId = Optional.of(MddTestDependencies.randomInstanceId()); + + Supplier<FileDownloader> fileDownloaderSupplier = + downloaderConfigurationType.fileDownloaderSupplier( + context, + controlExecutor, + DOWNLOAD_EXECUTOR, + fileStorage, + flags, + Optional.of(mockDownloadProgressMonitor), + instanceId); + BlockingFileDownloader blockingFileDownloader = + new BlockingFileDownloader(DOWNLOAD_EXECUTOR, fileDownloaderSupplier.get()); + MobileDataDownload mobileDataDownload = - MobileDataDownloadBuilder.newBuilder() - .setContext(context) - .setControlExecutor(CONTROL_EXECUTOR) - .setFileDownloaderSupplier(fileDownloaderSupplier) - .addFileGroupPopulator(testFileGroupPopulator) - .setTaskScheduler(Optional.of(mockTaskScheduler)) - .setDeltaDecoderOptional(Optional.absent()) - .setFileStorage(fileStorage) - .setNetworkUsageMonitor(mockNetworkUsageMonitor) - .setDownloadMonitorOptional(Optional.of(mockDownloadProgressMonitor)) - .setFlagsOptional(Optional.of(flags)) + builderForTest() + .setInstanceIdOptional(instanceId) + .setFileDownloaderSupplier(() -> blockingFileDownloader) .build(); - testFileGroupPopulator.refreshFileGroups(mobileDataDownload).get(); - mobileDataDownload - .downloadFileGroup( - DownloadFileGroupRequest.newBuilder() - .setGroupName(FILE_GROUP_NAME) - .setListenerOptional( - Optional.of( - new DownloadListener() { - @Override - public void onProgress(long currentSize) { - Log.i(TAG, "onProgress " + currentSize); - } + mobileDataDownload.clear().get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); - @Override - public void onComplete(ClientFileGroup clientFileGroup) { - Log.i(TAG, "onComplete " + clientFileGroup.getGroupName()); - } - })) - .build()) - .get(MAX_DOWNLOAD_FILE_GROUP_WAIT_TIME_SECS, TimeUnit.SECONDS); + // Add the filegroup, start downloading, then cancel while in progress. + TestFileGroupPopulator testFileGroupPopulator = new TestFileGroupPopulator(context); + testFileGroupPopulator + .refreshFileGroups(mobileDataDownload) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); - String debugString = mobileDataDownload.getDebugInfoAsString(); - Log.i(TAG, "MDD Lib dump:"); - for (String line : debugString.split("\n", -1)) { - Log.i(TAG, line); - } + ListenableFuture<ClientFileGroup> downloadFuture = + mobileDataDownload.downloadFileGroup( + DownloadFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()); + + blockingFileDownloader.finishDownloading(); // Unblocks blockingFileDownloader + blockingFileDownloader.waitForDelegateStarted(); // Waits until offroadDownloader starts + + // NOTE: add a little wait to allow Downloader's listeners to run. + Thread.sleep(/* millis= */ 200); + + downloadFuture.cancel(true /* may interrupt */); + + // NOTE: add a little wait to allow Downloader's listeners to run. + Thread.sleep(/* millis= */ 200); + + // Remove the filegroup. + ListenableFuture<Boolean> removeFuture = + mobileDataDownload.removeFileGroup( + RemoveFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()); + removeFuture.get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); + + // Add then try to download again. + blockingFileDownloader.resetState(); + blockingFileDownloader.finishDownloading(); // Unblocks blockingFileDownloader + + testFileGroupPopulator + .refreshFileGroups(mobileDataDownload) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); + downloadFuture = + mobileDataDownload.downloadFileGroup( + DownloadFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()); + + downloadFuture.get(MAX_DOWNLOAD_FILE_GROUP_WAIT_TIME_SECS, SECONDS); + + // The file should have downloaded as expected. ClientFileGroup clientFileGroup = mobileDataDownload .getFileGroup(GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) - .get(); + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); assertThat(clientFileGroup).isNotNull(); - assertThat(clientFileGroup.getGroupName()).isEqualTo(FILE_GROUP_NAME); assertThat(clientFileGroup.getFileCount()).isEqualTo(1); - - ClientFile clientFile = clientFileGroup.getFileList().get(0); - assertThat(clientFile.getFileId()).isEqualTo(FILE_ID); - Uri androidUri = Uri.parse(clientFile.getFileUri()); + Uri androidUri = Uri.parse(clientFileGroup.getFileList().get(0).getFileUri()); assertThat(fileStorage.fileSize(androidUri)).isEqualTo(FILE_SIZE); + } + + @Test + public void downloadDifferentGroupsWithSameFileTest() throws Exception { + Optional<String> instanceId = Optional.of(MddTestDependencies.randomInstanceId()); + MobileDataDownload mobileDataDownload = + builderForTest() + .setInstanceIdOptional(instanceId) + .setFileDownloaderSupplier( + () -> + new TestFileDownloader(TEST_DATA_RELATIVE_PATH, fileStorage, DOWNLOAD_EXECUTOR)) + .build(); + + DataFile.Builder dataFileBuilder = + DataFile.newBuilder() + .setUrlToDownload(TEST_DATA_URL) + .setChecksum(TEST_DATA_CHECKSUM) + .setByteSize(TEST_DATA_BYTE_SIZE); + DataFileGroup.Builder groupBuilder = DataFileGroup.newBuilder(); + + // Add all groups concurrently + ArrayList<ListenableFuture<Boolean>> addFutures = new ArrayList<>(); + for (int i = 0; i < 50; i++) { + String groupName = String.format("group%d", i); + String fileId = String.format("group%d_file", i); + + DataFile file = dataFileBuilder.setFileId(fileId).build(); + DataFileGroup group = + DataFileGroup.newBuilder().setGroupName(groupName).addFile(file).build(); + + addFutures.add( + mobileDataDownload.addFileGroup( + AddFileGroupRequest.newBuilder().setDataFileGroup(group).build())); + } + Futures.allAsList(addFutures).get(MAX_MULTI_MDD_API_WAIT_TIME_SECS, SECONDS); - mobileDataDownload.clear().get(); + // Start all downloads concurrently + ArrayList<ListenableFuture<ClientFileGroup>> downloadFutures = new ArrayList<>(); + for (int i = 0; i < 50; i++) { + String groupName = String.format("group%d", i); - switch (version) { - case V2: - // No-op + downloadFutures.add( + mobileDataDownload.downloadFileGroup( + DownloadFileGroupRequest.newBuilder().setGroupName(groupName).build())); } + List<ClientFileGroup> groups = + Futures.allAsList(downloadFutures).get(MAX_MULTI_MDD_API_WAIT_TIME_SECS, SECONDS); + + assertThat(groups).doesNotContain(null); } @Test - public void cancelDownload() throws Exception { - // In this test we will start a download and make sure that calling cancel on the returned - // future will cancel the download. - // We create a BlockingFileDownloader that allows the download to be blocked indefinitely. - // We also provide a delegate FileDownloader that attaches a FutureCallback to the internal - // download future and fail if the future is not cancelled. - BlockingFileDownloader blockingFileDownloader = - new BlockingFileDownloader( - listeningExecutorService, - new FileDownloader() { - @Override - public ListenableFuture<Void> startDownloading(DownloadRequest downloadRequest) { - ListenableFuture<Void> downloadTaskFuture = Futures.immediateVoidFuture(); - Futures.addCallback( - downloadTaskFuture, - new FutureCallback<Void>() { - @Override - public void onSuccess(Void result) { - // Should not get here since we will cancel the future. - fail(); - } - - @Override - public void onFailure(Throwable t) { - assertThat(downloadTaskFuture.isCancelled()).isTrue(); - - Log.i(TAG, "downloadTask is cancelled!"); - } - }, - listeningExecutorService); - return downloadTaskFuture; - } - }); - Supplier<FileDownloader> neverFinishDownloader = () -> blockingFileDownloader; + public void concurrentDownloads_withSameFile_withDifferentDownloadTransforms_completes( + @TestParameter boolean enableDedupByFileKey) throws Exception { + flags.enableFileDownloadDedupByFileKey = Optional.of(enableDedupByFileKey); - // Use never finish downloader to test whether the cancellation on the downloadFuture would - // cancel all the parent futures. - TestFileGroupPopulator testFileGroupPopulator = new TestFileGroupPopulator(context); + Optional<String> instanceId = Optional.of(MddTestDependencies.randomInstanceId()); MobileDataDownload mobileDataDownload = - MobileDataDownloadBuilder.newBuilder() - .setContext(context) - .setControlExecutor(CONTROL_EXECUTOR) - .setFileDownloaderSupplier(neverFinishDownloader) - .addFileGroupPopulator(testFileGroupPopulator) - .setTaskScheduler(Optional.of(mockTaskScheduler)) - .setDeltaDecoderOptional(Optional.absent()) - .setFileStorage(fileStorage) - .setNetworkUsageMonitor(mockNetworkUsageMonitor) - .setDownloadMonitorOptional(Optional.of(mockDownloadProgressMonitor)) - .setFlagsOptional(Optional.of(flags)) + builderForTest() + .setInstanceIdOptional(instanceId) + .setFileDownloaderSupplier( + () -> + new TestFileDownloader(TEST_DATA_RELATIVE_PATH, fileStorage, DOWNLOAD_EXECUTOR)) .build(); - testFileGroupPopulator.refreshFileGroups(mobileDataDownload).get(); + // Create two groups which share the same file, but have different download transforms + DataFileGroup groupWithoutTransform = + DataFileGroup.newBuilder() + .setGroupName("groupWithoutTransform") + .addFile( + DataFile.newBuilder() + .setFileId("file_no_transform") + .setUrlToDownload(TEST_DATA_URL) + .setChecksum(TEST_DATA_CHECKSUM) + .setByteSize(TEST_DATA_BYTE_SIZE)) + .build(); - // Now start to download the file group. - ListenableFuture<ClientFileGroup> downloadFileGroupFuture = - mobileDataDownload.downloadFileGroup( - DownloadFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()); + DataFileGroup groupWithTransform = + DataFileGroup.newBuilder() + .setGroupName("groupWithTransform") + .addFile( + DataFile.newBuilder() + .setFileId("file_no_transform") + .setUrlToDownload(TEST_DATA_COMPRESS_URL) + .setChecksum(TEST_DATA_CHECKSUM) + .setByteSize(TEST_DATA_BYTE_SIZE) + .setDownloadedFileChecksum(TEST_DATA_COMPRESS_CHECKSUM) + .setDownloadedFileByteSize(TEST_DATA_COMPRESS_BYTE_SIZE) + .setDownloadTransforms( + Transforms.newBuilder() + .addTransform( + Transform.newBuilder() + .setCompress( + TransformProto.CompressTransform.getDefaultInstance()) + .build()) + .build()) + .build()) + .build(); - // Note: we could have a race condition here between when we call the - // downloadFileGroupFuture.cancel and when the FileDownloader.startDownloading is executed. - // The following call will ensure that we will only call cancel on the downloadFileGroupFuture - // when the actual download has happened (the downloadTaskFuture). - // This will block until the downloadTaskFuture starts. - blockingFileDownloader.waitForDownloadStarted(); + // Add both groups, then attempt to download both concurrently + mobileDataDownload + .addFileGroup( + AddFileGroupRequest.newBuilder().setDataFileGroup(groupWithoutTransform).build()) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); + mobileDataDownload + .addFileGroup(AddFileGroupRequest.newBuilder().setDataFileGroup(groupWithTransform).build()) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); - // Cancel the downloadFileGroupFuture, it should cascade cancellation to downloadTaskFuture. - downloadFileGroupFuture.cancel(true /*may interrupt*/); + ListenableFuture<ClientFileGroup> downloadWithoutTransform = + mobileDataDownload.downloadFileGroup( + DownloadFileGroupRequest.newBuilder().setGroupName("groupWithoutTransform").build()); + ListenableFuture<ClientFileGroup> downloadWithTransform = + mobileDataDownload.downloadFileGroup( + DownloadFileGroupRequest.newBuilder().setGroupName("groupWithTransform").build()); - // Allow the download to continue and trigger our delegate FileDownloader. If the future isn't - // cancelled, the onSuccess callback should fail the test. - blockingFileDownloader.finishDownloading(); - blockingFileDownloader.waitForDownloadCompleted(); + List<ClientFileGroup> downloadedGroups = + Futures.allAsList(ImmutableList.of(downloadWithoutTransform, downloadWithTransform)) + .get(MAX_MULTI_MDD_API_WAIT_TIME_SECS, SECONDS); - assertThat(downloadFileGroupFuture.isCancelled()).isTrue(); + // Both groups are downloaded and both files point to the same on-device uri. + assertThat(downloadedGroups).doesNotContain(null); + assertThat(downloadedGroups.get(0).getFile(0).getFileUri()) + .isEqualTo(downloadedGroups.get(1).getFile(0).getFileUri()); + } - mobileDataDownload.clear().get(); + /** + * Returns MDD Builder with common dependencies set -- additional dependencies are added in each + * test as needed. + */ + private MobileDataDownloadBuilder builderForTest() { + + return MobileDataDownloadBuilder.newBuilder() + .setContext(context) + .setControlExecutor(controlExecutor) + .setFileStorage(fileStorage) + .setTaskScheduler(Optional.of(mockTaskScheduler)) + .setDeltaDecoderOptional(Optional.absent()) + .setNetworkUsageMonitor(mockNetworkUsageMonitor) + .setDownloadMonitorOptional(Optional.of(mockDownloadProgressMonitor)) + .setFlagsOptional(Optional.of(flags)); } } diff --git a/javatests/com/google/android/libraries/mobiledatadownload/DownloadFileIntegrationTest.java b/javatests/com/google/android/libraries/mobiledatadownload/DownloadFileIntegrationTest.java index e1b37a0..9a8625a 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/DownloadFileIntegrationTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/DownloadFileIntegrationTest.java @@ -16,15 +16,17 @@ package com.google.android.libraries.mobiledatadownload; import static com.google.android.libraries.mobiledatadownload.DownloadException.DownloadResultCode.ANDROID_DOWNLOADER_HTTP_ERROR; +import static com.google.android.libraries.mobiledatadownload.testing.MddTestDependencies.ExecutorType; import static com.google.common.truth.Truth.assertThat; +import static com.google.common.util.concurrent.Futures.immediateVoidFuture; import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import android.content.Context; import android.net.Uri; import androidx.test.core.app.ApplicationProvider; -import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.android.libraries.mobiledatadownload.downloader.DownloadConstraints; import com.google.android.libraries.mobiledatadownload.downloader.FileDownloader; import com.google.android.libraries.mobiledatadownload.downloader.offroad.dagger.downloader2.BaseFileDownloaderModule; @@ -42,11 +44,14 @@ import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; +import com.google.testing.junit.testparameterinjector.TestParameter; +import com.google.testing.junit.testparameterinjector.TestParameterInjector; import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; +import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -55,44 +60,51 @@ import org.mockito.Mock; import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; -@RunWith(AndroidJUnit4.class) +@RunWith(TestParameterInjector.class) public class DownloadFileIntegrationTest { - @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + @Rule(order = 1) + public final MockitoRule mocks = MockitoJUnit.rule(); private static final String TAG = "DownloadFileIntegrationTest"; + private static final long TIMEOUT_MS = 3000; + private static final int FILE_SIZE = 554; private static final String FILE_URL = "https://www.gstatic.com/suggest-dev/odws1_empty.jar"; private static final String DOES_NOT_EXIST_FILE_URL = "https://www.gstatic.com/non-existing/suggest-dev/not-exist.txt"; - // Note: Control Executor must not be a single thread executor. - private static final ListeningExecutorService CONTROL_EXECUTOR = - MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); - private static final ScheduledExecutorService DOWNLOAD_EXECUTOR = - Executors.newScheduledThreadPool(2); + private static final ListeningScheduledExecutorService DOWNLOAD_EXECUTOR = + MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(2)); private static final Context context = ApplicationProvider.getApplicationContext(); - private MobileDataDownload mobileDataDownload; - private final Uri destinationFileUri = AndroidUri.builder(context).setModule("mdd").setRelativePath("file_1").build(); + private final FakeTimeSource clock = new FakeTimeSource(); + private final TestFlags flags = new TestFlags(); + private MobileDataDownload mobileDataDownload; private DownloadProgressMonitor downloadProgressMonitor; private SynchronousFileStorage fileStorage; + private Supplier<FileDownloader> fileDownloaderSupplier; - private final FakeTimeSource clock = new FakeTimeSource(); + private ListeningExecutorService controlExecutor; @Mock private SingleFileDownloadListener mockDownloadListener; @Mock private NetworkUsageMonitor mockNetworkUsageMonitor; - private final TestFlags flags = new TestFlags(); + @TestParameter ExecutorType controlExecutorType; @Before public void setUp() throws Exception { - downloadProgressMonitor = new DownloadProgressMonitor(clock, CONTROL_EXECUTOR); + // Set a default behavior for the download listener. + when(mockDownloadListener.onComplete()).thenReturn(immediateVoidFuture()); + + controlExecutor = controlExecutorType.executor(); + + downloadProgressMonitor = new DownloadProgressMonitor(clock, controlExecutor); fileStorage = new SynchronousFileStorage( @@ -105,23 +117,31 @@ public class DownloadFileIntegrationTest { BaseFileDownloaderModule.createOffroad2FileDownloader( context, DOWNLOAD_EXECUTOR, - CONTROL_EXECUTOR, + controlExecutor, fileStorage, new SharedPreferencesDownloadMetadata( - context.getSharedPreferences("downloadmetadata", 0), CONTROL_EXECUTOR), + context.getSharedPreferences("downloadmetadata", 0), controlExecutor), Optional.of(downloadProgressMonitor), /* urlEngineOptional= */ Optional.absent(), /* exceptionHandlerOptional= */ Optional.absent(), /* authTokenProviderOptional= */ Optional.absent(), +// /* cookieJarSupplierOptional= */ Optional.absent(), /* trafficTag= */ Optional.absent(), flags); } + @After + public void tearDown() throws Exception { + if (fileStorage.exists(destinationFileUri)) { + fileStorage.deleteFile(destinationFileUri); + } + } + @Test public void downloadFile_success() throws Exception { - assertThat(fileStorage.exists(destinationFileUri)).isFalse(); + mobileDataDownload = builderForTest().build(); - mobileDataDownload = getMobileDataDownload(fileDownloaderSupplier); + assertThat(fileStorage.exists(destinationFileUri)).isFalse(); SingleFileDownloadRequest downloadRequest = SingleFileDownloadRequest.newBuilder() @@ -141,15 +161,15 @@ public class DownloadFileIntegrationTest { // Verify the downloadListener is called. // Sleep for 1 sec to wait for the Future's callback to finish. - Thread.sleep(/*millis=*/ 1000); + Thread.sleep(/* millis= */ 1000); verify(mockDownloadListener).onComplete(); } @Test public void downloadFile_failure() throws Exception { - assertThat(fileStorage.exists(destinationFileUri)).isFalse(); + mobileDataDownload = builderForTest().build(); - mobileDataDownload = getMobileDataDownload(fileDownloaderSupplier); + assertThat(fileStorage.exists(destinationFileUri)).isFalse(); // Trying to download doesn't exist URL. SingleFileDownloadRequest downloadRequest = @@ -171,16 +191,17 @@ public class DownloadFileIntegrationTest { // Verify the downloadListener is called. // Sleep for 1 sec to wait for the Future's callback to finish. - Thread.sleep(/*millis=*/ 1000); + Thread.sleep(/* millis= */ 1000); verify(mockDownloadListener).onFailure(any(DownloadException.class)); } @Test public void downloadFile_cancel() throws Exception { - // Reinitialize downloader with a BlockingFileDownloader to ensure download remains in progress - // until it is cancelled. - BlockingFileDownloader blockingFileDownloader = new BlockingFileDownloader(CONTROL_EXECUTOR); - mobileDataDownload = getMobileDataDownload(() -> blockingFileDownloader); + // Use a BlockingFileDownloader to ensure download remains in progress until it is cancelled. + BlockingFileDownloader blockingFileDownloader = new BlockingFileDownloader(DOWNLOAD_EXECUTOR); + + mobileDataDownload = + builderForTest().setFileDownloaderSupplier(() -> blockingFileDownloader).build(); SingleFileDownloadRequest downloadRequest = SingleFileDownloadRequest.newBuilder() @@ -205,16 +226,18 @@ public class DownloadFileIntegrationTest { blockingFileDownloader.resetState(); } - private MobileDataDownload getMobileDataDownload( - Supplier<FileDownloader> fileDownloaderSupplier) { + /** + * Returns MDD Builder with common dependencies set -- additional dependencies are added in each + * test as needed. + */ + private MobileDataDownloadBuilder builderForTest() { return MobileDataDownloadBuilder.newBuilder() .setContext(context) - .setControlExecutor(CONTROL_EXECUTOR) + .setControlExecutor(controlExecutor) .setFileDownloaderSupplier(fileDownloaderSupplier) .setFileStorage(fileStorage) .setDownloadMonitorOptional(Optional.of(downloadProgressMonitor)) .setNetworkUsageMonitor(mockNetworkUsageMonitor) - .setFlagsOptional(Optional.of(flags)) - .build(); + .setFlagsOptional(Optional.of(flags)); } } diff --git a/javatests/com/google/android/libraries/mobiledatadownload/DownloadFileTest.java b/javatests/com/google/android/libraries/mobiledatadownload/DownloadFileTest.java index 2a8ed40..2305858 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/DownloadFileTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/DownloadFileTest.java @@ -33,6 +33,7 @@ import com.google.android.libraries.mobiledatadownload.downloader.DownloadReques import com.google.android.libraries.mobiledatadownload.downloader.FileDownloader; import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; import com.google.android.libraries.mobiledatadownload.file.backends.AndroidFileBackend; +import com.google.android.libraries.mobiledatadownload.foreground.ForegroundDownloadKey; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.android.libraries.mobiledatadownload.monitor.DownloadProgressMonitor; import com.google.android.libraries.mobiledatadownload.monitor.NetworkUsageMonitor; @@ -181,7 +182,7 @@ public final class DownloadFileTest { mobileDataDownload = getMobileDataDownload( () -> mockFileDownloader, - /* foregroundDownloadServiceClassOptional = */ Optional.absent(), + /* foregroundDownloadServiceClassOptional= */ Optional.absent(), Optional.of(mockDownloadMonitor)); singleFileDownloadRequest = @@ -235,7 +236,7 @@ public final class DownloadFileTest { mobileDataDownload = getMobileDataDownload( createSuccessfulFileDownloaderSupplier(), - /* foregroundDownloadServiceClassOptional = */ Optional.absent(), + /* foregroundDownloadServiceClassOptional= */ Optional.absent(), Optional.of(downloadProgressMonitor)); singleFileDownloadRequest = @@ -262,7 +263,7 @@ public final class DownloadFileTest { mobileDataDownload = getMobileDataDownload( createFailingFileDownloaderSupplier(downloadException), - /* foregroundDownloadServiceClassOptional = */ Optional.absent(), + /* foregroundDownloadServiceClassOptional= */ Optional.absent(), Optional.of(downloadProgressMonitor)); singleFileDownloadRequest = @@ -338,7 +339,7 @@ public final class DownloadFileTest { mobileDataDownload = getMobileDataDownload( () -> mockFileDownloader, - /* foregroundDownloadServiceClassOptional = */ Optional.absent(), + /* foregroundDownloadServiceClassOptional= */ Optional.absent(), Optional.of(downloadProgressMonitor)); // Without foreground service, download call should fail with IllegalStateException @@ -358,7 +359,7 @@ public final class DownloadFileTest { getMobileDataDownload( () -> mockFileDownloader, Optional.of(this.getClass()), - /* downloadProgressMonitorOptional = */ Optional.absent()); + /* downloadProgressMonitorOptional= */ Optional.absent()); // Without monitor, download call should fail with IllegalStateException ListenableFuture<Void> downloadFuture = @@ -565,10 +566,15 @@ public final class DownloadFileTest { // Use BlockingFileDownloader to control when the download will finish. mobileDataDownload = getMobileDataDownload(() -> blockingFileDownloader); + ForegroundDownloadKey foregroundDownloadKey = + ForegroundDownloadKey.ofSingleFile(DESTINATION_FILE_URI); + ListenableFuture<Void> downloadFuture = mobileDataDownload.downloadFileWithForegroundService(singleFileDownloadRequest); - mobileDataDownload.cancelForegroundDownload(DESTINATION_FILE_URI.toString()); + blockingFileDownloader.waitForDownloadStarted(); + + mobileDataDownload.cancelForegroundDownload(foregroundDownloadKey.toString()); awaitAllExecutorsIdle(); diff --git a/javatests/com/google/android/libraries/mobiledatadownload/ImportFilesIntegrationTest.java b/javatests/com/google/android/libraries/mobiledatadownload/ImportFilesIntegrationTest.java index dc5f1ea..2748641 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/ImportFilesIntegrationTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/ImportFilesIntegrationTest.java @@ -20,6 +20,7 @@ import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopul import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_ID; import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_SIZE; import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_URL; +import static com.google.android.libraries.mobiledatadownload.testing.MddTestDependencies.ExecutorType; import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertThrows; import static org.junit.Assert.fail; @@ -28,7 +29,6 @@ import android.content.Context; import android.net.Uri; import android.os.Environment; import androidx.test.core.app.ApplicationProvider; -import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.android.libraries.mobiledatadownload.DownloadException.DownloadResultCode; import com.google.android.libraries.mobiledatadownload.downloader.DownloadRequest; import com.google.android.libraries.mobiledatadownload.downloader.FileDownloader; @@ -62,8 +62,11 @@ import com.google.mobiledatadownload.ClientConfigProto.ClientFile; import com.google.mobiledatadownload.ClientConfigProto.ClientFileGroup; import com.google.mobiledatadownload.ClientConfigProto.ClientFileGroup.Status; import com.google.mobiledatadownload.DownloadConfigProto.DataFile; +import com.google.mobiledatadownload.DownloadConfigProto.DataFile.ChecksumType; import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; import com.google.protobuf.ByteString; +import com.google.testing.junit.testparameterinjector.TestParameter; +import com.google.testing.junit.testparameterinjector.TestParameterInjector; import java.io.IOException; import java.io.InputStream; import java.util.concurrent.ExecutionException; @@ -79,7 +82,7 @@ import org.mockito.Mock; import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; -@RunWith(AndroidJUnit4.class) +@RunWith(TestParameterInjector.class) public final class ImportFilesIntegrationTest { @Rule public final MockitoRule mocks = MockitoJUnit.rule(); @@ -88,18 +91,18 @@ public final class ImportFilesIntegrationTest { private static final String TEST_DATA_ABSOLUTE_PATH = Environment.getExternalStorageDirectory() - + "/googletest/test_runfiles/google3/third_party/java_src/android_libs/mobiledatadownload/javatests/com/google/android/libraries/mobiledatadownload/testdata/"; + + "/googletest/test_runfiles/third_party/java_src/android_libs/mobiledatadownload/javatests/com/google/android/libraries/mobiledatadownload/testdata/"; - // Note: Control Executor must not be a single thread executor. - private static final ListeningExecutorService CONTROL_EXECUTOR = - MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); private static final ScheduledExecutorService DOWNLOAD_EXECUTOR = Executors.newScheduledThreadPool(2); private static final String FILE_ID_1 = "test-file-1"; private static final Uri FILE_URI_1 = Uri.parse( - FileUri.builder().setPath(TEST_DATA_ABSOLUTE_PATH + "odws1_empty").build().toString()); + FileUri.builder() + .setPath(TEST_DATA_ABSOLUTE_PATH + "odws1_empty.jar") + .build() + .toString()); private static final String FILE_CHECKSUM_1 = "a1cba9d87b1440f41ce9e7da38c43e1f6bd7d5df"; private static final String FILE_URL_1 = "inlinefile:sha1:" + FILE_CHECKSUM_1; private static final int FILE_SIZE_1 = 554; @@ -129,21 +132,37 @@ public final class ImportFilesIntegrationTest { .setChecksum(FILE_CHECKSUM_2) .build(); + private static final long BUILD_ID = 10; + private static final String VARIANT_ID = "default"; + private static final String FILE_ID_3 = "empty-inline-file"; + private static final String FILE_URL_3 = + String.format("inlinefile:buildId:%s:variantId:%s", BUILD_ID, VARIANT_ID); + private static final DataFile EMPTY_INLINE_FILE = + DataFile.newBuilder() + .setFileId(FILE_ID_3) + .setChecksumType(ChecksumType.NONE) + .setUrlToDownload(FILE_URL_3) + .build(); + private static final Context context = ApplicationProvider.getApplicationContext(); + private final TestFlags flags = new TestFlags(); + @Mock private TaskScheduler mockTaskScheduler; @Mock private NetworkUsageMonitor mockNetworkUsageMonitor; @Mock private DownloadProgressMonitor mockDownloadProgressMonitor; private FakeFileBackend fakeFileBackend; private SynchronousFileStorage fileStorage; + private Supplier<FileDownloader> multiSchemeFileDownloaderSupplier; private MobileDataDownload mobileDataDownload; + private ListeningExecutorService controlExecutor; private FileSource inlineFileSource1; private FileSource inlineFileSource2; - private final TestFlags flags = new TestFlags(); + @TestParameter ExecutorType controlExecutorType; @Before public void setUp() throws Exception { @@ -155,37 +174,42 @@ public final class ImportFilesIntegrationTest { /* transforms= */ ImmutableList.of(new CompressTransform()), /* monitors= */ ImmutableList.of(mockNetworkUsageMonitor, mockDownloadProgressMonitor)); - Supplier<FileDownloader> fileDownloaderSupplier = + // Set up inline file sources + try (InputStream fileStream1 = fileStorage.open(FILE_URI_1, ReadStreamOpener.create()); + InputStream fileStream2 = fileStorage.open(FILE_URI_2, ReadStreamOpener.create())) { + inlineFileSource1 = FileSource.ofByteString(ByteString.readFrom(fileStream1)); + inlineFileSource2 = FileSource.ofByteString(ByteString.readFrom(fileStream2)); + } + + controlExecutor = controlExecutorType.executor(); + + Supplier<FileDownloader> httpsFileDownloaderSupplier = () -> BaseFileDownloaderModule.createOffroad2FileDownloader( context, DOWNLOAD_EXECUTOR, - CONTROL_EXECUTOR, + controlExecutor, fileStorage, new SharedPreferencesDownloadMetadata( - context.getSharedPreferences("downloadmetadata", 0), CONTROL_EXECUTOR), + context.getSharedPreferences("downloadmetadata", 0), controlExecutor), Optional.of(mockDownloadProgressMonitor), /* urlEngineOptional= */ Optional.absent(), /* exceptionHandlerOptional= */ Optional.absent(), /* authTokenProviderOptional= */ Optional.absent(), +// /* cookieJarSupplierOptional= */ Optional.absent(), /* trafficTag= */ Optional.absent(), flags); Supplier<FileDownloader> inlineFileDownloaderSupplier = () -> new InlineFileDownloader(fileStorage, DOWNLOAD_EXECUTOR); + multiSchemeFileDownloaderSupplier = () -> MultiSchemeFileDownloader.builder() - .addScheme("https", fileDownloaderSupplier.get()) + .addScheme("https", httpsFileDownloaderSupplier.get()) .addScheme("inlinefile", inlineFileDownloaderSupplier.get()) .build(); - - // Set up inline file sources - try (InputStream fileStream1 = fileStorage.open(FILE_URI_1, ReadStreamOpener.create()); - InputStream fileStream2 = fileStorage.open(FILE_URI_2, ReadStreamOpener.create())) { - inlineFileSource1 = FileSource.ofByteString(ByteString.readFrom(fileStream1)); - inlineFileSource2 = FileSource.ofByteString(ByteString.readFrom(fileStream2)); - } + flags.enableDownloadStageExperimentIdPropagation = Optional.of(true); } @After @@ -198,7 +222,7 @@ public final class ImportFilesIntegrationTest { @Test public void importFiles_performsImport() throws Exception { - createMobileDataDownload(multiSchemeFileDownloaderSupplier); + mobileDataDownload = builderForTest().build(); DataFileGroup fileGroupWithInlineFile = DataFileGroup.newBuilder() @@ -244,7 +268,7 @@ public final class ImportFilesIntegrationTest { @Test public void importFiles_whenImportingMultipleFiles_performsImport() throws Exception { - createMobileDataDownload(multiSchemeFileDownloaderSupplier); + mobileDataDownload = builderForTest().build(); DataFileGroup fileGroupWithInlineFile = DataFileGroup.newBuilder() @@ -306,7 +330,8 @@ public final class ImportFilesIntegrationTest { return multiSchemeFileDownloaderSupplier.get().startDownloading(request); } }); - createMobileDataDownload(() -> blockingFileDownloader); + mobileDataDownload = + builderForTest().setFileDownloaderSupplier(() -> blockingFileDownloader).build(); DataFileGroup fileGroup1WithInlineFile = DataFileGroup.newBuilder() @@ -365,7 +390,9 @@ public final class ImportFilesIntegrationTest { blockingFileDownloader.finishDownloading(); // Wait for both futures to complete - Futures.whenAllSucceed(importFuture1, importFuture2).call(() -> null, CONTROL_EXECUTOR).get(); + Futures.whenAllSucceed(importFuture1, importFuture2) + .call(() -> null, MoreExecutors.directExecutor()) + .get(); // Assert that the resulting group is downloaded and contains a reference to on device file ClientFileGroup importResult1 = @@ -397,7 +424,7 @@ public final class ImportFilesIntegrationTest { @Test public void importFiles_whenNewInlineFileSpecified_importsAndStoresFile() throws Exception { - createMobileDataDownload(multiSchemeFileDownloaderSupplier); + mobileDataDownload = builderForTest().build(); DataFileGroup fileGroupWithOneInlineFile = DataFileGroup.newBuilder() @@ -448,7 +475,7 @@ public final class ImportFilesIntegrationTest { @Test public void importFiles_whenNewInlineFileAddedToPendingGroup_importsAndStoresFile() throws Exception { - createMobileDataDownload(multiSchemeFileDownloaderSupplier); + mobileDataDownload = builderForTest().build(); DataFileGroup fileGroupWithStandardFile = DataFileGroup.newBuilder() @@ -522,7 +549,7 @@ public final class ImportFilesIntegrationTest { @Test public void importFiles_toNonExistentDataFileGroup_fails() throws Exception { - createMobileDataDownload(multiSchemeFileDownloaderSupplier); + mobileDataDownload = builderForTest().build(); FileSource inlineFileSource = FileSource.ofByteString(ByteString.copyFromUtf8("TEST_CONTENT")); @@ -547,7 +574,7 @@ public final class ImportFilesIntegrationTest { @Test public void importFiles_whenMismatchedVersion_failToImport() throws Exception { - createMobileDataDownload(multiSchemeFileDownloaderSupplier); + mobileDataDownload = builderForTest().build(); DataFileGroup fileGroupWithInlineFile = DataFileGroup.newBuilder() @@ -586,7 +613,7 @@ public final class ImportFilesIntegrationTest { @Test public void importFiles_whenImportFails_doesNotWriteUpdatedMetadata() throws Exception { - createMobileDataDownload(multiSchemeFileDownloaderSupplier); + mobileDataDownload = builderForTest().build(); // Create initial file group to import DataFileGroup initialFileGroup = @@ -681,7 +708,8 @@ public final class ImportFilesIntegrationTest { } }); - createMobileDataDownload(() -> blockingFileDownloader); + mobileDataDownload = + builderForTest().setFileDownloaderSupplier(() -> blockingFileDownloader).build(); DataFileGroup fileGroup1WithInlineFile = DataFileGroup.newBuilder() @@ -783,7 +811,8 @@ public final class ImportFilesIntegrationTest { } }); - createMobileDataDownload(() -> blockingFileDownloader); + mobileDataDownload = + builderForTest().setFileDownloaderSupplier(() -> blockingFileDownloader).build(); DataFileGroup fileGroupWithInlineFile = DataFileGroup.newBuilder() @@ -816,7 +845,7 @@ public final class ImportFilesIntegrationTest { // wait for the file downloader to be invoked before performing the cancel. blockingFileDownloader.waitForDownloadStarted(); - importFilesFuture.cancel(/* mayInterruptIfRunning = */ true); + importFilesFuture.cancel(/* mayInterruptIfRunning= */ true); // Allow the download to continue and trigger our delegate FileDownloader. If the future isn't // cancelled, the onSuccess callback should fail the test. @@ -828,18 +857,101 @@ public final class ImportFilesIntegrationTest { mobileDataDownload.clear().get(); } - private void createMobileDataDownload(Supplier<FileDownloader> fileDownloaderSupplier) { - mobileDataDownload = - MobileDataDownloadBuilder.newBuilder() - .setContext(context) - .setControlExecutor(CONTROL_EXECUTOR) - .setFileDownloaderSupplier(fileDownloaderSupplier) - .setTaskScheduler(Optional.of(mockTaskScheduler)) - .setDeltaDecoderOptional(Optional.absent()) - .setFileStorage(fileStorage) - .setNetworkUsageMonitor(mockNetworkUsageMonitor) - .setDownloadMonitorOptional(Optional.of(mockDownloadProgressMonitor)) - .setFlagsOptional(Optional.of(flags)) + @Test + public void importFiles_emptyInlineFileImport_withExperimentInfo() throws Exception { + mobileDataDownload = builderForTest().build(); + + DataFileGroup fileGroupWithInlineFile = + DataFileGroup.newBuilder() + .setBuildId(BUILD_ID) + .setStaleLifetimeSecs(0) + .setVariantId(VARIANT_ID) + .setGroupName(FILE_GROUP_NAME) + .addFile(EMPTY_INLINE_FILE) .build(); + + // Ensure that we add the file group successfully. + assertThat( + mobileDataDownload + .addFileGroup( + AddFileGroupRequest.newBuilder() + .setDataFileGroup(fileGroupWithInlineFile) + .build()) + .get()) + .isTrue(); + + // Use getFileGroupsByFilter to get the file group. + ImmutableList<ClientFileGroup> allFileGroups = + mobileDataDownload + .getFileGroupsByFilter( + GetFileGroupsByFilterRequest.newBuilder() + .setGroupNameOptional(Optional.of(FILE_GROUP_NAME)) + .build()) + .get(); + + // Assert that the resulting group is pending. + assertThat(allFileGroups.get(0).getStatus()).isEqualTo(Status.PENDING); + + // Perform the import. + mobileDataDownload + .importFiles( + ImportFilesRequest.newBuilder() + .setGroupName(FILE_GROUP_NAME) + .setBuildId(BUILD_ID) + .setVariantId(VARIANT_ID) + .setInlineFileMap( + ImmutableMap.of(FILE_ID_3, FileSource.ofByteString(ByteString.EMPTY))) + .build()) + .get(); + + // Assert that the resulting group is downloaded and contains a reference to on device file. + ClientFileGroup importResult = + mobileDataDownload + .getFileGroup(GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) + .get(); + Uri importFileUri = Uri.parse(importResult.getFile(0).getFileUri()); + + // Verify if correct DOWNLOADED stage experiment Ids are attached. + assertThat(importResult).isNotNull(); + assertThat(importResult.getGroupName()).isEqualTo(FILE_GROUP_NAME); + assertThat(importResult.getFileCount()).isEqualTo(1); + assertThat(importResult.getStatus()).isEqualTo(Status.DOWNLOADED); + assertThat(fileStorage.exists(importFileUri)).isTrue(); + + // Remove the filegroup which has been downloaded. + mobileDataDownload + .removeFileGroup(RemoveFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) + .get(); + + importResult = + mobileDataDownload + .getFileGroup(GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) + .get(); + + // Assert no active filegroup. + assertThat(importResult).isNull(); + + // Run MDD maintenance task. + mobileDataDownload.handleTask(TaskScheduler.MAINTENANCE_PERIODIC_TASK).get(); + + // Assert file removed from file storage. + assertThat(fileStorage.exists(importFileUri)).isFalse(); + } + + /** + * Returns MDD Builder with common dependencies set -- additional dependencies are added in each + * test as needed. + */ + private MobileDataDownloadBuilder builderForTest() { + return MobileDataDownloadBuilder.newBuilder() + .setContext(context) + .setControlExecutor(controlExecutor) + .setFileDownloaderSupplier(multiSchemeFileDownloaderSupplier) + .setTaskScheduler(Optional.of(mockTaskScheduler)) + .setDeltaDecoderOptional(Optional.absent()) + .setFileStorage(fileStorage) + .setNetworkUsageMonitor(mockNetworkUsageMonitor) + .setDownloadMonitorOptional(Optional.of(mockDownloadProgressMonitor)) + .setFlagsOptional(Optional.of(flags)); } } diff --git a/javatests/com/google/android/libraries/mobiledatadownload/MddGarbageCollectionWithAndroidSharingIntegrationTest.java b/javatests/com/google/android/libraries/mobiledatadownload/MddGarbageCollectionWithAndroidSharingIntegrationTest.java index 3d73e04..bc00cc3 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/MddGarbageCollectionWithAndroidSharingIntegrationTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/MddGarbageCollectionWithAndroidSharingIntegrationTest.java @@ -15,16 +15,19 @@ */ package com.google.android.libraries.mobiledatadownload; +import static com.google.android.libraries.mobiledatadownload.testing.MddTestDependencies.ExecutorType; import static com.google.common.truth.Truth.assertThat; import static java.util.concurrent.TimeUnit.DAYS; import static java.util.concurrent.TimeUnit.SECONDS; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import android.app.blob.BlobStoreManager; import android.content.Context; import android.net.Uri; import android.util.Log; import androidx.test.core.app.ApplicationProvider; -import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.android.libraries.mobiledatadownload.downloader.FileDownloader; import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; import com.google.android.libraries.mobiledatadownload.file.backends.AndroidFileBackend; @@ -46,6 +49,11 @@ import com.google.mobiledatadownload.ClientConfigProto.ClientFile; import com.google.mobiledatadownload.ClientConfigProto.ClientFileGroup; import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; import com.google.mobiledatadownload.DownloadConfigProto.DownloadConditions.DeviceNetworkPolicy; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; +import com.google.mobiledatadownload.LogProto.MddLogData; +import com.google.testing.junit.testparameterinjector.TestParameter; +import com.google.testing.junit.testparameterinjector.TestParameterInjector; +import java.util.List; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import org.junit.After; @@ -53,11 +61,12 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; -@RunWith(AndroidJUnit4.class) +@RunWith(TestParameterInjector.class) public final class MddGarbageCollectionWithAndroidSharingIntegrationTest { private static final String TAG = "MddGarbageCollectionWithAndroidSharingIntegrationTest"; private static final int MAX_DOWNLOAD_FILE_GROUP_WAIT_TIME_SECS = 300; @@ -65,9 +74,6 @@ public final class MddGarbageCollectionWithAndroidSharingIntegrationTest { private static final String TEST_DATA_RELATIVE_PATH = "third_party/java_src/android_libs/mobiledatadownload/javatests/com/google/android/libraries/mobiledatadownload/testdata/"; - // Note: Control Executor must not be a single thread executor. - private static final ListeningExecutorService CONTROL_EXECUTOR = - MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); private static final ScheduledExecutorService DOWNLOAD_EXECUTOR = Executors.newScheduledThreadPool(2); @@ -87,20 +93,40 @@ public final class MddGarbageCollectionWithAndroidSharingIntegrationTest { @Mock private Logger mockLogger; private SynchronousFileStorage fileStorage; + private BlobStoreManager blobStoreManager; private MobileDataDownload mobileDataDownload; + private Supplier<FileDownloader> fileDownloaderSupplier; + private ListeningExecutorService controlExecutor; private final TestFlags flags = new TestFlags(); - @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + @Rule(order = 1) + public final MockitoRule mocks = MockitoJUnit.rule(); + + @TestParameter ExecutorType controlExecutorType; @Before public void setUp() throws Exception { + + // cl/439051122 created a temporary FALSE override targeted to ASGA devices. This test suite + // relies on garbage collection being enabled to test the metadata state transistions, but + // all_on testing doesn't respect diversion criteria in the launch. + // + // So we temporarily force it on to bypass the launch so the tests can rely on expected + // behavior. + // TODO(b/226551373): remove these overrides once AsgaDisableMddLibGcLaunch is turned down + flags.mddEnableGarbageCollection = Optional.of(true); + flags.mddAndroidSharingSampleInterval = Optional.of(1); + flags.mddDefaultSampleInterval = Optional.of(1); + BlobStoreBackend blobStoreBackend = new BlobStoreBackend(context); blobStoreManager = (BlobStoreManager) context.getSystemService(Context.BLOB_STORE_SERVICE); + controlExecutor = controlExecutorType.executor(); + fileStorage = new SynchronousFileStorage( /* backends= */ ImmutableList.of( @@ -109,26 +135,13 @@ public final class MddGarbageCollectionWithAndroidSharingIntegrationTest { new JavaFileBackend()), /* transforms= */ ImmutableList.of(new CompressTransform()), /* monitors= */ ImmutableList.of(mockNetworkUsageMonitor, mockDownloadProgressMonitor)); - Supplier<FileDownloader> fileDownloaderSupplier = + + fileDownloaderSupplier = () -> new TestFileDownloader( TEST_DATA_RELATIVE_PATH, fileStorage, MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR)); - - mobileDataDownload = - MobileDataDownloadBuilder.newBuilder() - .setContext(context) - .setControlExecutor(CONTROL_EXECUTOR) - .setFileDownloaderSupplier(fileDownloaderSupplier) - .setTaskScheduler(Optional.of(mockTaskScheduler)) - .setDeltaDecoderOptional(Optional.absent()) - .setFileStorage(fileStorage) - .setNetworkUsageMonitor(mockNetworkUsageMonitor) - .setDownloadMonitorOptional(Optional.of(mockDownloadProgressMonitor)) - .setLoggerOptional(Optional.of(mockLogger)) - .setFlagsOptional(Optional.of(flags)) - .build(); } @After @@ -182,6 +195,7 @@ public final class MddGarbageCollectionWithAndroidSharingIntegrationTest { @Test public void deletesStaleGroups_staleLifetimeZero() throws Exception { + mobileDataDownload = builderForTest().build(); Uri androidUri = BlobUri.builder(context).setBlobParameters(FILE_ANDROID_SHARING_CHECKSUM_1).build(); assertThat(fileStorage.exists(androidUri)).isFalse(); @@ -234,10 +248,46 @@ public final class MddGarbageCollectionWithAndroidSharingIntegrationTest { assertThat(blobStoreManager.getLeasedBlobs()).isEmpty(); // Verify logging events. + + ArgumentCaptor<MddLogData> logDataCaptor = ArgumentCaptor.forClass(MddLogData.class); + // 1050 is the tag number for MddClientEvent.Code.EVENT_CODE_UNSPECIFIED. + verify(mockLogger).log(logDataCaptor.capture(), /* eventCode= */ eq(1050)); + List<MddLogData> logData = logDataCaptor.getAllValues(); + assertThat(logData).hasSize(1); + + DataDownloadFileGroupStats dataDownloadFileGroupStats = + logData.get(0).getDataDownloadFileGroupStats(); + DataDownloadFileGroupStats staleGroupExpired = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(fileGroup.getGroupName()) + .setFileGroupVersionNumber(fileGroup.getFileGroupVersionNumber()) + .setBuildId(0) + .setVariantId("") + .build(); + assertThat(dataDownloadFileGroupStats).isEqualTo(staleGroupExpired); + + logDataCaptor = ArgumentCaptor.forClass(MddLogData.class); + // 1084 is the tag number for MddClientEvent.Code.EVENT_CODE_UNSPECIFIED; + // Called once for every released lease. + verify(mockLogger).log(logDataCaptor.capture(), /* eventCode= */ eq(1084)); + + logDataCaptor = ArgumentCaptor.forClass(MddLogData.class); + // 1051 is the tag number for MddClientEvent.Code.EVENT_CODE_UNSPECIFIED. + // It's logged once by mobileDataDownload.maintenance() and three times in the + // ExpirationHandler, once when the file metadata is deleted, once when the lease is released + // and once when the temporary local copy of the shared file is deleted. + verify(mockLogger, times(4)).log(logDataCaptor.capture(), /* eventCode= */ eq(1051)); + logData = logDataCaptor.getAllValues(); + assertThat(logData).hasSize(4); + + Void metadafileDeleted = null; + Void fileReleased = null; + Void fileDeleted = null; } @Test public void deletesStaleGroups_staleLifetimeTwoDays() throws Exception { + mobileDataDownload = builderForTest().build(); Uri androidUri = BlobUri.builder(context).setBlobParameters(FILE_ANDROID_SHARING_CHECKSUM_1).build(); assertThat(fileStorage.exists(androidUri)).isFalse(); @@ -299,10 +349,46 @@ public final class MddGarbageCollectionWithAndroidSharingIntegrationTest { assertThat(blobStoreManager.getLeasedBlobs()).isEmpty(); // Verify logging events. + + ArgumentCaptor<MddLogData> logDataCaptor = ArgumentCaptor.forClass(MddLogData.class); + // 1050 is the tag number for MddClientEvent.Code.EVENT_CODE_UNSPECIFIED. + verify(mockLogger).log(logDataCaptor.capture(), /* eventCode= */ eq(1050)); + List<MddLogData> logData = logDataCaptor.getAllValues(); + assertThat(logData).hasSize(1); + + DataDownloadFileGroupStats dataDownloadFileGroupStats = + logData.get(0).getDataDownloadFileGroupStats(); + DataDownloadFileGroupStats staleGroupExpired = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(fileGroup.getGroupName()) + .setFileGroupVersionNumber(fileGroup.getFileGroupVersionNumber()) + .setBuildId(0) + .setVariantId("") + .build(); + assertThat(dataDownloadFileGroupStats).isEqualTo(staleGroupExpired); + + logDataCaptor = ArgumentCaptor.forClass(MddLogData.class); + // 1084 is the tag number for MddClientEvent.Code.EVENT_CODE_UNSPECIFIED; + // Called once for every released lease. + verify(mockLogger).log(logDataCaptor.capture(), /* eventCode= */ eq(1084)); + + logDataCaptor = ArgumentCaptor.forClass(MddLogData.class); + // 1051 is the tag number for MddClientEvent.Code.EVENT_CODE_UNSPECIFIED. + // It's logged once every time mobileDataDownload.maintenance() is called and three times in the + // ExpirationHandler, once when the file metadata is deleted, once when the lease is released + // and once when the temporary local copy of the shared file is deleted. + verify(mockLogger, times(5)).log(logDataCaptor.capture(), /* eventCode= */ eq(1051)); + logData = logDataCaptor.getAllValues(); + assertThat(logData).hasSize(5); + + Void metadafileDeleted = null; + Void fileReleased = null; + Void fileDeleted = null; } @Test public void deletesExpiredGroups() throws Exception { + mobileDataDownload = builderForTest().build(); Uri androidUri = BlobUri.builder(context).setBlobParameters(FILE_ANDROID_SHARING_CHECKSUM_1).build(); assertThat(fileStorage.exists(androidUri)).isFalse(); @@ -357,5 +443,58 @@ public final class MddGarbageCollectionWithAndroidSharingIntegrationTest { // Verify logging events. + ArgumentCaptor<MddLogData> logDataCaptor = ArgumentCaptor.forClass(MddLogData.class); + // 1049 is the tag number for MddClientEvent.Code.EVENT_CODE_UNSPECIFIED. + verify(mockLogger).log(logDataCaptor.capture(), /* eventCode= */ eq(1049)); + + List<MddLogData> logData = logDataCaptor.getAllValues(); + assertThat(logData).hasSize(1); + + DataDownloadFileGroupStats dataDownloadFileGroupStats = + logData.get(0).getDataDownloadFileGroupStats(); + DataDownloadFileGroupStats groupExpired = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(fileGroup.getGroupName()) + .setFileGroupVersionNumber(fileGroup.getFileGroupVersionNumber()) + .setBuildId(0) + .setVariantId("") + .build(); + assertThat(dataDownloadFileGroupStats).isEqualTo(groupExpired); + + logDataCaptor = ArgumentCaptor.forClass(MddLogData.class); + // 1084 is the tag number for MddClientEvent.Code.EVENT_CODE_UNSPECIFIED; + // Called once for every released lease. + verify(mockLogger).log(logDataCaptor.capture(), /* eventCode= */ eq(1084)); + + logDataCaptor = ArgumentCaptor.forClass(MddLogData.class); + // 1051 is the tag number for MddClientEvent.Code.EVENT_CODE_UNSPECIFIED. + // It's logged once every time mobileDataDownload.maintenance() is called and three times in the + // ExpirationHandler, once when the file metadata is deleted, once when the lease is + // released and once when the temporary local copy of the shared file is deleted. + verify(mockLogger, times(5)).log(logDataCaptor.capture(), /* eventCode= */ eq(1051)); + logData = logDataCaptor.getAllValues(); + assertThat(logData).hasSize(5); + + Void metadafileDeleted = null; + Void fileReleased = null; + Void fileDeleted = null; + } + + /** + * Returns MDD Builder with common dependencies set -- additional dependencies are added in each + * test as needed. + */ + private MobileDataDownloadBuilder builderForTest() { + return MobileDataDownloadBuilder.newBuilder() + .setContext(context) + .setControlExecutor(controlExecutor) + .setFileDownloaderSupplier(fileDownloaderSupplier) + .setTaskScheduler(Optional.of(mockTaskScheduler)) + .setDeltaDecoderOptional(Optional.absent()) + .setFileStorage(fileStorage) + .setNetworkUsageMonitor(mockNetworkUsageMonitor) + .setDownloadMonitorOptional(Optional.of(mockDownloadProgressMonitor)) + .setLoggerOptional(Optional.of(mockLogger)) + .setFlagsOptional(Optional.of(flags)); } } diff --git a/javatests/com/google/android/libraries/mobiledatadownload/MobileDataDownloadIntegrationTest.java b/javatests/com/google/android/libraries/mobiledatadownload/MobileDataDownloadIntegrationTest.java index c94532e..ab03cd7 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/MobileDataDownloadIntegrationTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/MobileDataDownloadIntegrationTest.java @@ -21,18 +21,21 @@ import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopul import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_ID; import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_SIZE; import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_URL; +import static com.google.android.libraries.mobiledatadownload.testing.MddTestDependencies.ExecutorType; import static com.google.android.libraries.mobiledatadownload.tracing.TracePropagation.propagateCallable; import static com.google.common.truth.Truth.assertThat; import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.Assert.assertThrows; import static org.junit.Assert.fail; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import android.accounts.Account; import android.content.Context; import android.net.Uri; import android.util.Log; import androidx.test.core.app.ApplicationProvider; -import androidx.test.runner.AndroidJUnit4; import com.google.android.libraries.mobiledatadownload.account.AccountUtil; import com.google.android.libraries.mobiledatadownload.downloader.FileDownloader; import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; @@ -54,42 +57,52 @@ import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.mobiledatadownload.ClientConfigProto.ClientFile; import com.google.mobiledatadownload.ClientConfigProto.ClientFileGroup; import com.google.mobiledatadownload.DownloadConfigProto.DataFile; -import com.google.mobiledatadownload.DownloadConfigProto.DataFile.ChecksumType; import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; -import com.google.mobiledatadownload.DownloadConfigProto.DownloadConditions; import com.google.mobiledatadownload.DownloadConfigProto.DownloadConditions.DeviceNetworkPolicy; +import com.google.mobiledatadownload.LogEnumsProto.MddDownloadResult; +import com.google.mobiledatadownload.LogProto.MddDownloadResultLog; +import com.google.mobiledatadownload.LogProto.MddLogData; import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeoutException; +import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; -@RunWith(AndroidJUnit4.class) +// NOTE: TestParameterInjector is preferred for parameterized tests, but it has a API +// level constraint of >= 24 while MDD has a constraint of >= 16. To prevent basic regressions, run +// this test using junit's Parameterized TestRunner, which supports all API levels. +@RunWith(Parameterized.class) public class MobileDataDownloadIntegrationTest { private static final String TAG = "MobileDataDownloadIntegrationTest"; private static final int MAX_HANDLE_TASK_WAIT_TIME_SECS = 300; + private static final int MAX_MDD_API_WAIT_TIME_SECS = 5; private static final String TEST_DATA_RELATIVE_PATH = "third_party/java_src/android_libs/mobiledatadownload/javatests/com/google/android/libraries/mobiledatadownload/testdata/"; - // Note: Control Executor must not be a single thread executor. - private static final ListeningExecutorService CONTROL_EXECUTOR = - MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); - private static final ScheduledExecutorService DOWNLOAD_EXECUTOR = - Executors.newScheduledThreadPool(2); + private static final ListeningScheduledExecutorService DOWNLOAD_EXECUTOR = + MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(2)); private static final Context context = ApplicationProvider.getApplicationContext(); private final NetworkUsageMonitor networkUsageMonitor = @@ -103,31 +116,52 @@ public class MobileDataDownloadIntegrationTest { private final TestFlags flags = new TestFlags(); + private ListeningExecutorService controlExecutor; + + private MobileDataDownload mobileDataDownload; + @Mock private Logger mockLogger; @Mock private TaskScheduler mockTaskScheduler; @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + @Parameter public ExecutorType controlExecutorType; + + @Parameters + public static Collection<Object[]> data() { + return Arrays.asList( + new Object[][] { + {ExecutorType.SINGLE_THREADED}, {ExecutorType.MULTI_THREADED}, + }); + } + @Before public void setUp() throws Exception { + flags.enableZipFolder = Optional.of(true); + + controlExecutor = controlExecutorType.executor(); + } + + @After + public void tearDown() throws Exception { + mobileDataDownload.clear().get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); } @Test public void download_success_fileGroupDownloaded() throws Exception { - MobileDataDownload mobileDataDownload = - getMobileDataDownloadAfterDownload( - () -> - new TestFileDownloader( - TEST_DATA_RELATIVE_PATH, - fileStorage, - MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR)), - new TestFileGroupPopulator(context)); + mobileDataDownload = + builderForTest() + .setFileDownloaderSupplier( + () -> + new TestFileDownloader( + TEST_DATA_RELATIVE_PATH, + fileStorage, + MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR))) + .addFileGroupPopulator(new TestFileGroupPopulator(context)) + .build(); - // This will trigger refreshing of FileGroupPopulators and downloading. - mobileDataDownload - .handleTask(TaskScheduler.CELLULAR_CHARGING_PERIODIC_TASK) - .get(MAX_HANDLE_TASK_WAIT_TIME_SECS, SECONDS); + waitForHandleTask(); String debugString = mobileDataDownload.getDebugInfoAsString(); Log.i(TAG, "MDD Lib dump:"); @@ -135,10 +169,8 @@ public class MobileDataDownloadIntegrationTest { Log.i(TAG, line); } - ClientFileGroup clientFileGroup = - getAndVerifyClientFileGroup(mobileDataDownload, FILE_GROUP_NAME, 1); + ClientFileGroup clientFileGroup = getAndVerifyClientFileGroup(FILE_GROUP_NAME, 1); verifyClientFile(clientFileGroup.getFileList().get(0), FILE_ID, FILE_SIZE); - mobileDataDownload.clear().get(); } @Test @@ -164,75 +196,84 @@ public class MobileDataDownloadIntegrationTest { })); }; - MobileDataDownload mobileDataDownload = - getMobileDataDownloadBuilder( + mobileDataDownload = + builderForTest() + .setFileDownloaderSupplier( () -> new TestFileDownloader( TEST_DATA_RELATIVE_PATH, fileStorage, - MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR)), - new TestFileGroupPopulator(context)) + MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR))) + .addFileGroupPopulator(new TestFileGroupPopulator(context)) .setCustomFileGroupValidatorOptional(Optional.of(validator)) .build(); - // This will trigger refreshing of FileGroupPopulators and downloading. - mobileDataDownload - .handleTask(TaskScheduler.CELLULAR_CHARGING_PERIODIC_TASK) - .get(MAX_HANDLE_TASK_WAIT_TIME_SECS, SECONDS); + waitForHandleTask(); ClientFileGroup clientFileGroup = mobileDataDownload .getFileGroup(GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) - .get(); + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); verifyClientFile(clientFileGroup.getFileList().get(0), FILE_ID, FILE_SIZE); - - mobileDataDownload.clear().get(); } @Test public void download_success_maintenanceLogsNetworkUsage() throws Exception { flags.networkStatsLoggingSampleInterval = Optional.of(1); - MobileDataDownload mobileDataDownload = - getMobileDataDownload( - () -> - new TestFileDownloader( - TEST_DATA_RELATIVE_PATH, - fileStorage, - MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR)), - new TestFileGroupPopulator(context)); + mobileDataDownload = + builderForTest() + .setFileDownloaderSupplier( + () -> + new TestFileDownloader( + TEST_DATA_RELATIVE_PATH, + fileStorage, + MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR))) + .addFileGroupPopulator(new TestFileGroupPopulator(context)) + .build(); - // This will trigger refreshing of FileGroupPopulators and downloading. - mobileDataDownload - .handleTask(TaskScheduler.CELLULAR_CHARGING_PERIODIC_TASK) - .get(MAX_HANDLE_TASK_WAIT_TIME_SECS, SECONDS); + waitForHandleTask(); // This should flush the logs from NetworkLogger. mobileDataDownload .handleTask(TaskScheduler.MAINTENANCE_PERIODIC_TASK) .get(MAX_HANDLE_TASK_WAIT_TIME_SECS, SECONDS); - ClientFileGroup clientFileGroup = - getAndVerifyClientFileGroup(mobileDataDownload, FILE_GROUP_NAME, 1); + ClientFileGroup clientFileGroup = getAndVerifyClientFileGroup(FILE_GROUP_NAME, 1); verifyClientFile(clientFileGroup.getFileList().get(0), FILE_ID, FILE_SIZE); - mobileDataDownload.clear().get(); + ArgumentCaptor<MddLogData> logDataCaptor = ArgumentCaptor.forClass(MddLogData.class); + // 1056 is the tag number for MddClientEvent.Code.EVENT_CODE_UNSPECIFIED. + verify(mockLogger, times(1)).log(logDataCaptor.capture(), /* eventCode= */ eq(1056)); + + List<MddLogData> logDataList = logDataCaptor.getAllValues(); + assertThat(logDataList).hasSize(1); + MddLogData logData = logDataList.get(0); + + Void mddNetworkStats = null; + + // Network status changes depending on emulator: + boolean isCellular = NetworkUsageMonitor.isCellular(context); } @Test public void corrupted_files_detectedDuringMaintenance() throws Exception { flags.mddDefaultSampleInterval = Optional.of(1); - MobileDataDownload mobileDataDownload = - getMobileDataDownloadAfterDownload( - () -> - new TestFileDownloader( - TEST_DATA_RELATIVE_PATH, - fileStorage, - MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR)), - new TestFileGroupPopulator(context)); - ClientFileGroup clientFileGroup = - getAndVerifyClientFileGroup(mobileDataDownload, FILE_GROUP_NAME, 1); + mobileDataDownload = + builderForTest() + .setFileDownloaderSupplier( + () -> + new TestFileDownloader( + TEST_DATA_RELATIVE_PATH, + fileStorage, + MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR))) + .addFileGroupPopulator(new TestFileGroupPopulator(context)) + .build(); + + waitForHandleTask(); + + ClientFileGroup clientFileGroup = getAndVerifyClientFileGroup(FILE_GROUP_NAME, 1); fileStorage.open( Uri.parse(clientFileGroup.getFile(0).getFileUri()), WriteStringOpener.create("c0rrupt3d")); @@ -247,29 +288,31 @@ public class MobileDataDownloadIntegrationTest { .get(MAX_HANDLE_TASK_WAIT_TIME_SECS, SECONDS); // Re-load the file group since the on-disk URIs will have changed. - clientFileGroup = getAndVerifyClientFileGroup(mobileDataDownload, FILE_GROUP_NAME, 1); + clientFileGroup = getAndVerifyClientFileGroup(FILE_GROUP_NAME, 1); assertThat( fileStorage.open( Uri.parse(clientFileGroup.getFile(0).getFileUri()), ReadStringOpener.create())) .isNotEqualTo("c0rrupt3d"); - - mobileDataDownload.clear().get(); } @Test public void delete_files_detectedDuringMaintenance() throws Exception { flags.mddDefaultSampleInterval = Optional.of(1); - MobileDataDownload mobileDataDownload = - getMobileDataDownloadAfterDownload( - () -> - new TestFileDownloader( - TEST_DATA_RELATIVE_PATH, - fileStorage, - MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR)), - new TestFileGroupPopulator(context)); - ClientFileGroup clientFileGroup = - getAndVerifyClientFileGroup(mobileDataDownload, FILE_GROUP_NAME, 1); + mobileDataDownload = + builderForTest() + .setFileDownloaderSupplier( + () -> + new TestFileDownloader( + TEST_DATA_RELATIVE_PATH, + fileStorage, + MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR))) + .addFileGroupPopulator(new TestFileGroupPopulator(context)) + .build(); + + waitForHandleTask(); + + ClientFileGroup clientFileGroup = getAndVerifyClientFileGroup(FILE_GROUP_NAME, 1); fileStorage.deleteFile(Uri.parse(clientFileGroup.getFile(0).getFileUri())); // Bad file is detected during maintenance. @@ -283,29 +326,24 @@ public class MobileDataDownloadIntegrationTest { .get(MAX_HANDLE_TASK_WAIT_TIME_SECS, SECONDS); // Re-load the file group since the on-disk URIs will have changed. - clientFileGroup = getAndVerifyClientFileGroup(mobileDataDownload, FILE_GROUP_NAME, 1); + clientFileGroup = getAndVerifyClientFileGroup(FILE_GROUP_NAME, 1); assertThat(fileStorage.exists(Uri.parse(clientFileGroup.getFile(0).getFileUri()))).isTrue(); - - mobileDataDownload.clear().get(); } @Test public void remove_withAccount_fileGroupRemains() throws Exception { - Supplier<FileDownloader> fileDownloaderSupplier = - () -> - new TestFileDownloader( - TEST_DATA_RELATIVE_PATH, - fileStorage, - MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR)); - - MobileDataDownload mobileDataDownload = - getMobileDataDownloadAfterDownload( - fileDownloaderSupplier, new TestFileGroupPopulator(context)); + mobileDataDownload = + builderForTest() + .setFileDownloaderSupplier( + () -> + new TestFileDownloader( + TEST_DATA_RELATIVE_PATH, + fileStorage, + MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR))) + .addFileGroupPopulator(new TestFileGroupPopulator(context)) + .build(); - // This will trigger refreshing of FileGroupPopulators and downloading. - mobileDataDownload - .handleTask(TaskScheduler.CELLULAR_CHARGING_PERIODIC_TASK) - .get(MAX_HANDLE_TASK_WAIT_TIME_SECS, SECONDS); + waitForHandleTask(); // Remove the file group with account doesn't change anything, because the test group is not // associated with any account. @@ -318,67 +356,166 @@ public class MobileDataDownloadIntegrationTest { .setGroupName(FILE_GROUP_NAME) .setAccountOptional(Optional.of(account)) .build()) - .get()) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS)) .isTrue(); - ClientFileGroup clientFileGroup = - getAndVerifyClientFileGroup(mobileDataDownload, FILE_GROUP_NAME, 1); + ClientFileGroup clientFileGroup = getAndVerifyClientFileGroup(FILE_GROUP_NAME, 1); verifyClientFile(clientFileGroup.getFileList().get(0), FILE_ID, FILE_SIZE); - mobileDataDownload.clear().get(); } @Test public void remove_withoutAccount_fileGroupRemoved() throws Exception { - Supplier<FileDownloader> fileDownloaderSupplier = - () -> - new TestFileDownloader( - TEST_DATA_RELATIVE_PATH, - fileStorage, - MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR)); - - MobileDataDownload mobileDataDownload = - getMobileDataDownloadAfterDownload( - fileDownloaderSupplier, new TestFileGroupPopulator(context)); + mobileDataDownload = + builderForTest() + .setFileDownloaderSupplier( + () -> + new TestFileDownloader( + TEST_DATA_RELATIVE_PATH, + fileStorage, + MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR))) + .addFileGroupPopulator(new TestFileGroupPopulator(context)) + .build(); - // This will trigger refreshing of FileGroupPopulators and downloading. - mobileDataDownload - .handleTask(TaskScheduler.CELLULAR_CHARGING_PERIODIC_TASK) - .get(MAX_HANDLE_TASK_WAIT_TIME_SECS, SECONDS); + waitForHandleTask(); // Remove the file group will make the file group not accessible from clients. assertThat( mobileDataDownload .removeFileGroup( RemoveFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) - .get()) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS)) .isTrue(); ClientFileGroup clientFileGroup = mobileDataDownload .getFileGroup(GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) - .get(); + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); assertThat(clientFileGroup).isNull(); - mobileDataDownload.clear().get(); + } + + @Test + public void removeFileGroupsByFilter_removesMatchingGroups() throws Exception { + mobileDataDownload = + builderForTest() + .setFileDownloaderSupplier( + () -> + new TestFileDownloader( + TEST_DATA_RELATIVE_PATH, + fileStorage, + MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR))) + .build(); + + // Remove All Groups to clear state + mobileDataDownload + .removeFileGroupsByFilter(RemoveFileGroupsByFilterRequest.newBuilder().build()) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); + + // Tear down: remove remaining group to prevent cross test errors + mobileDataDownload + .removeFileGroupsByFilter(RemoveFileGroupsByFilterRequest.newBuilder().build()) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); + } + + @Test + public void removeFileGroupsByFilter_whenAccountSpecified_removesMatchingAccountDependentGroups() + throws Exception { + mobileDataDownload = + builderForTest() + .setFileDownloaderSupplier( + () -> + new TestFileDownloader( + TEST_DATA_RELATIVE_PATH, + fileStorage, + MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR))) + .build(); + + // Remove all groups + mobileDataDownload + .removeFileGroupsByFilter(RemoveFileGroupsByFilterRequest.newBuilder().build()) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); + + // Setup account + Account account = AccountUtil.create("name", "google"); + + // Setup two groups, 1 with account and 1 without an account + DataFileGroup fileGroupWithoutAccount = + TestFileGroupPopulator.createDataFileGroup( + FILE_GROUP_NAME, + context.getPackageName(), + new String[] {FILE_ID}, + new int[] {FILE_SIZE}, + new String[] {FILE_CHECKSUM}, + new String[] {FILE_URL}, + DeviceNetworkPolicy.DOWNLOAD_ON_ANY_NETWORK) + .toBuilder() + .build(); + DataFileGroup fileGroupWithAccount = + fileGroupWithoutAccount.toBuilder().setGroupName(FILE_GROUP_NAME + "_2").build(); + + // Add both groups to MDD + mobileDataDownload + .addFileGroup( + AddFileGroupRequest.newBuilder().setDataFileGroup(fileGroupWithoutAccount).build()) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); + mobileDataDownload + .addFileGroup( + AddFileGroupRequest.newBuilder() + .setDataFileGroup(fileGroupWithAccount) + .setAccountOptional(Optional.of(account)) + .build()) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); + + // Verify that both groups are present + assertThat( + mobileDataDownload + .getFileGroupsByFilter( + GetFileGroupsByFilterRequest.newBuilder().setIncludeAllGroups(true).build()) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS)) + .hasSize(2); + + // Remove file groups with given account and source + mobileDataDownload + .removeFileGroupsByFilter( + RemoveFileGroupsByFilterRequest.newBuilder() + .setAccountOptional(Optional.of(account)) + .build()) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); + + // Check that only account-independent group remains + ImmutableList<ClientFileGroup> remainingGroups = + mobileDataDownload + .getFileGroupsByFilter( + GetFileGroupsByFilterRequest.newBuilder().setIncludeAllGroups(true).build()) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); + assertThat(remainingGroups).hasSize(1); + assertThat(remainingGroups.get(0).getGroupName()).isEqualTo(FILE_GROUP_NAME); + + // Tear down: remove remaining group to prevent cross test errors + mobileDataDownload + .removeFileGroupsByFilter(RemoveFileGroupsByFilterRequest.newBuilder().build()) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); } @Test public void removeFileGroupsByFilter_whenAccountNotSpecified_removesMatchingAccountIndependentGroups() throws Exception { - Supplier<FileDownloader> fileDownloaderSupplier = - () -> - new TestFileDownloader( - TEST_DATA_RELATIVE_PATH, - fileStorage, - MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR)); + mobileDataDownload = + builderForTest() + .setFileDownloaderSupplier( + () -> + new TestFileDownloader( + TEST_DATA_RELATIVE_PATH, + fileStorage, + MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR))) + .build(); - MobileDataDownload mobileDataDownload = - getMobileDataDownload(fileDownloaderSupplier, unused -> Futures.immediateVoidFuture()); + waitForHandleTask(); // Remove all groups mobileDataDownload .removeFileGroupsByFilter(RemoveFileGroupsByFilterRequest.newBuilder().build()) - .get(); + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); // Setup account Account account = AccountUtil.create("name", "google"); @@ -402,34 +539,34 @@ public class MobileDataDownloadIntegrationTest { mobileDataDownload .addFileGroup( AddFileGroupRequest.newBuilder().setDataFileGroup(fileGroupWithoutAccount).build()) - .get(); + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); mobileDataDownload .addFileGroup( AddFileGroupRequest.newBuilder() .setDataFileGroup(fileGroupWithAccount) .setAccountOptional(Optional.of(account)) .build()) - .get(); + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); // Verify that both groups are present assertThat( mobileDataDownload .getFileGroupsByFilter( GetFileGroupsByFilterRequest.newBuilder().setIncludeAllGroups(true).build()) - .get()) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS)) .hasSize(2); // Remove file groups with given source only mobileDataDownload .removeFileGroupsByFilter(RemoveFileGroupsByFilterRequest.newBuilder().build()) - .get(); + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); // Check that only account-dependent group remains ImmutableList<ClientFileGroup> remainingGroups = mobileDataDownload .getFileGroupsByFilter( GetFileGroupsByFilterRequest.newBuilder().setIncludeAllGroups(true).build()) - .get(); + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); assertThat(remainingGroups).hasSize(1); assertThat(remainingGroups.get(0).getGroupName()).isEqualTo(FILE_GROUP_NAME + "_2"); @@ -439,22 +576,25 @@ public class MobileDataDownloadIntegrationTest { RemoveFileGroupsByFilterRequest.newBuilder() .setAccountOptional(Optional.of(account)) .build()) - .get(); + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); } @Test public void download_failure_throwsDownloadException() throws Exception { flags.mddDefaultSampleInterval = Optional.of(1); - Supplier<FileDownloader> fileDownloaderSupplier = - () -> - new TestFileDownloader( - TEST_DATA_RELATIVE_PATH, - fileStorage, - MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR)); + mobileDataDownload = + builderForTest() + .setFileDownloaderSupplier( + () -> + new TestFileDownloader( + TEST_DATA_RELATIVE_PATH, + fileStorage, + MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR))) + .addFileGroupPopulator(new TestFileGroupPopulator(context)) + .build(); - MobileDataDownload mobileDataDownload = - getMobileDataDownload(fileDownloaderSupplier, new TestFileGroupPopulator(context)); + waitForHandleTask(); DataFileGroup dataFileGroup = TestFileGroupPopulator.createDataFileGroup( @@ -473,7 +613,7 @@ public class MobileDataDownloadIntegrationTest { mobileDataDownload .addFileGroup( AddFileGroupRequest.newBuilder().setDataFileGroup(dataFileGroup).build()) - .get()) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS)) .isTrue(); ListenableFuture<ClientFileGroup> downloadFuture = @@ -494,15 +634,16 @@ public class MobileDataDownloadIntegrationTest { public void download_failure_logsEvent() throws Exception { flags.mddDefaultSampleInterval = Optional.of(1); - Supplier<FileDownloader> fileDownloaderSupplier = - () -> - new TestFileDownloader( - TEST_DATA_RELATIVE_PATH, - fileStorage, - MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR)); - - MobileDataDownload mobileDataDownload = - getMobileDataDownload(fileDownloaderSupplier, new TestFileGroupPopulator(context)); + mobileDataDownload = + builderForTest() + .setFileDownloaderSupplier( + () -> + new TestFileDownloader( + TEST_DATA_RELATIVE_PATH, + fileStorage, + MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR))) + .addFileGroupPopulator(new TestFileGroupPopulator(context)) + .build(); DataFileGroup dataFileGroup = TestFileGroupPopulator.createDataFileGroup( @@ -521,7 +662,7 @@ public class MobileDataDownloadIntegrationTest { mobileDataDownload .addFileGroup( AddFileGroupRequest.newBuilder().setDataFileGroup(dataFileGroup).build()) - .get()) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS)) .isTrue(); ListenableFuture<ClientFileGroup> downloadFuture = @@ -529,23 +670,53 @@ public class MobileDataDownloadIntegrationTest { DownloadFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()); assertThrows(ExecutionException.class, downloadFuture::get); + + if (controlExecutorType.equals(ExecutorType.SINGLE_THREADED)) { + // Single-threaded executor step requires some time to allow logging to finish. + // TODO: Investigate whether TestingTaskBarrier can be used here to wait for + // executor become idle. + Thread.sleep(500); + } + + ArgumentCaptor<MddLogData> logDataCaptor = ArgumentCaptor.forClass(MddLogData.class); + // 1068 is the tag number for MddClientEvent.Code.DATA_DOWNLOAD_RESULT_LOG. + verify(mockLogger, times(2)).log(logDataCaptor.capture(), /* eventCode= */ eq(1068)); + + List<MddLogData> logData = logDataCaptor.getAllValues(); + assertThat(logData).hasSize(2); + + MddDownloadResultLog downloadResultLog1 = logData.get(0).getMddDownloadResultLog(); + MddDownloadResultLog downloadResultLog2 = logData.get(1).getMddDownloadResultLog(); + assertThat(downloadResultLog1.getResult()).isEqualTo(MddDownloadResult.Code.INSECURE_URL_ERROR); + assertThat(downloadResultLog1.getDataDownloadFileGroupStats().getFileGroupName()) + .isEqualTo(FILE_GROUP_NAME); + assertThat(downloadResultLog1.getDataDownloadFileGroupStats().getOwnerPackage()) + .isEqualTo(context.getPackageName()); + assertThat(downloadResultLog2.getResult()) + .isEqualTo(MddDownloadResult.Code.ANDROID_DOWNLOADER_HTTP_ERROR); + assertThat(downloadResultLog2.getDataDownloadFileGroupStats().getFileGroupName()) + .isEqualTo(FILE_GROUP_NAME); + assertThat(downloadResultLog2.getDataDownloadFileGroupStats().getOwnerPackage()) + .isEqualTo(context.getPackageName()); } @Test public void download_zipFile_unzippedAfterDownload() throws Exception { - Supplier<FileDownloader> fileDownloaderSupplier = - () -> - new TestFileDownloader( - TEST_DATA_RELATIVE_PATH, - fileStorage, - MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR)); - - MobileDataDownload mobileDataDownload = - getMobileDataDownloadAfterDownload( - fileDownloaderSupplier, new ZipFolderFileGroupPopulator(context)); + mobileDataDownload = + builderForTest() + .setFileDownloaderSupplier( + () -> + new TestFileDownloader( + TEST_DATA_RELATIVE_PATH, + fileStorage, + MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR))) + .addFileGroupPopulator(new ZipFolderFileGroupPopulator(context)) + .build(); + + waitForHandleTask(); + ClientFileGroup clientFileGroup = - getAndVerifyClientFileGroup( - mobileDataDownload, ZipFolderFileGroupPopulator.FILE_GROUP_NAME, 3); + getAndVerifyClientFileGroup(ZipFolderFileGroupPopulator.FILE_GROUP_NAME, 3); for (ClientFile clientFile : clientFileGroup.getFileList()) { if ("/zip1.txt".equals(clientFile.getFileId())) { @@ -562,12 +733,12 @@ public class MobileDataDownloadIntegrationTest { @Test public void download_cancelDuringDownload_downloadCancelled() throws Exception { - BlockingFileDownloader blockingFileDownloader = new BlockingFileDownloader(CONTROL_EXECUTOR); + BlockingFileDownloader blockingFileDownloader = new BlockingFileDownloader(DOWNLOAD_EXECUTOR); Supplier<FileDownloader> fakeFileDownloaderSupplier = () -> blockingFileDownloader; - MobileDataDownload mobileDataDownload = - getMobileDataDownload(fakeFileDownloaderSupplier, new TestFileGroupPopulator(context)); + mobileDataDownload = + builderForTest().setFileDownloaderSupplier(fakeFileDownloaderSupplier).build(); // Register the file group and trigger download. mobileDataDownload @@ -583,7 +754,7 @@ public class MobileDataDownloadIntegrationTest { new String[] {FILE_URL}, DeviceNetworkPolicy.DOWNLOAD_ON_ANY_NETWORK)) .build()) - .get(); + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); ListenableFuture<ClientFileGroup> downloadFuture = mobileDataDownload.downloadFileGroup( DownloadFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()); @@ -595,7 +766,7 @@ public class MobileDataDownloadIntegrationTest { // Now remove the file group from MDD, which would cancel any ongoing download. mobileDataDownload .removeFileGroup(RemoveFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) - .get(); + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); // Now let the download future finish. blockingFileDownloader.finishDownloading(); @@ -614,25 +785,25 @@ public class MobileDataDownloadIntegrationTest { @Test public void download_twoStepDownload_targetFileDownloaded() throws Exception { - Supplier<FileDownloader> fileDownloaderSupplier = - () -> - new TestFileDownloader( - TEST_DATA_RELATIVE_PATH, - fileStorage, - MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR)); - - MobileDataDownload mobileDataDownload = - getMobileDataDownload(fileDownloaderSupplier, new TwoStepPopulator(context, fileStorage)); + mobileDataDownload = + builderForTest() + .setFileDownloaderSupplier( + () -> + new TestFileDownloader( + TEST_DATA_RELATIVE_PATH, + fileStorage, + MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR))) + .addFileGroupPopulator(new TwoStepPopulator(context, fileStorage)) + .build(); // Add step1 file group to MDD. DataFileGroup step1FileGroup = - createDataFileGroup( + TestFileGroupPopulator.createDataFileGroup( "step1-file-group", context.getPackageName(), new String[] {"step1_id"}, new int[] {57}, new String[] {""}, - new ChecksumType[] {ChecksumType.NONE}, new String[] {"https://www.gstatic.com/icing/idd/sample_group/step1.txt"}, DeviceNetworkPolicy.DOWNLOAD_ON_ANY_NETWORK); @@ -649,24 +820,26 @@ public class MobileDataDownloadIntegrationTest { // step2-file-group and it was downloaded too in one cycle (one call of handleTask). // Verify step1-file-group. - ClientFileGroup clientFileGroup = - getAndVerifyClientFileGroup(mobileDataDownload, "step1-file-group", 1); + ClientFileGroup clientFileGroup = getAndVerifyClientFileGroup("step1-file-group", 1); verifyClientFile(clientFileGroup.getFile(0), "step1_id", 57); // Verify step2-file-group. - clientFileGroup = getAndVerifyClientFileGroup(mobileDataDownload, "step2-file-group", 1); + clientFileGroup = getAndVerifyClientFileGroup("step2-file-group", 1); verifyClientFile(clientFileGroup.getFile(0), "step2_id", 13); - - mobileDataDownload.clear().get(); } @Test public void download_relativeFilePaths_createsSymlinks() throws Exception { AndroidUriAdapter adapter = AndroidUriAdapter.forContext(context); - MobileDataDownload mobileDataDownload = - getMobileDataDownload( - () -> new TestFileDownloader(TEST_DATA_RELATIVE_PATH, fileStorage, CONTROL_EXECUTOR), - new TestFileGroupPopulator(context)); + mobileDataDownload = + builderForTest() + .setFileDownloaderSupplier( + () -> + new TestFileDownloader( + TEST_DATA_RELATIVE_PATH, + fileStorage, + MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR))) + .build(); DataFileGroup fileGroup = DataFileGroup.newBuilder() @@ -686,21 +859,21 @@ public class MobileDataDownloadIntegrationTest { mobileDataDownload .addFileGroup(AddFileGroupRequest.newBuilder().setDataFileGroup(fileGroup).build()) - .get(); + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); mobileDataDownload .downloadFileGroup( DownloadFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) - .get(); + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); - // verify symlink structure + // verify symlink structure, we can't get access to the full internal file uri, but we can tell + // the start of it Uri expectedFileUri = DirectoryUtil.getBaseDownloadDirectory(context, Optional.absent()) .buildUpon() .appendPath(DirectoryUtil.MDD_STORAGE_SYMLINKS) .appendPath(DirectoryUtil.MDD_STORAGE_ALL_GOOGLE_APPS) .appendPath(FILE_GROUP_NAME) - .appendPath("relative_path") .build(); // we can't get access to the full internal target file uri, but we know the start of it Uri expectedStartTargetUri = @@ -713,7 +886,7 @@ public class MobileDataDownloadIntegrationTest { ClientFileGroup clientFileGroup = mobileDataDownload .getFileGroup(GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) - .get(); + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); Uri fileUri = Uri.parse(clientFileGroup.getFile(0).getFileUri()); Uri targetUri = @@ -721,7 +894,7 @@ public class MobileDataDownloadIntegrationTest { .fromAbsolutePath(readlink(adapter.toFile(fileUri).getAbsolutePath())) .build(); - assertThat(fileUri).isEqualTo(expectedFileUri); + assertThat(fileUri.toString()).contains(expectedFileUri.toString()); assertThat(targetUri.toString()).contains(expectedStartTargetUri.toString()); assertThat(fileStorage.exists(fileUri)).isTrue(); assertThat(fileStorage.exists(targetUri)).isTrue(); @@ -729,10 +902,15 @@ public class MobileDataDownloadIntegrationTest { @Test public void remove_relativeFilePaths_removesSymlinks() throws Exception { - MobileDataDownload mobileDataDownload = - getMobileDataDownload( - () -> new TestFileDownloader(TEST_DATA_RELATIVE_PATH, fileStorage, CONTROL_EXECUTOR), - new TestFileGroupPopulator(context)); + mobileDataDownload = + builderForTest() + .setFileDownloaderSupplier( + () -> + new TestFileDownloader( + TEST_DATA_RELATIVE_PATH, + fileStorage, + MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR))) + .build(); DataFileGroup fileGroup = DataFileGroup.newBuilder() @@ -752,17 +930,17 @@ public class MobileDataDownloadIntegrationTest { mobileDataDownload .addFileGroup(AddFileGroupRequest.newBuilder().setDataFileGroup(fileGroup).build()) - .get(); + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); mobileDataDownload .downloadFileGroup( DownloadFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) - .get(); + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); ClientFileGroup clientFileGroup = mobileDataDownload .getFileGroup(GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) - .get(); + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); Uri fileUri = Uri.parse(clientFileGroup.getFile(0).getFileUri()); @@ -771,71 +949,82 @@ public class MobileDataDownloadIntegrationTest { mobileDataDownload .removeFileGroup(RemoveFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) - .get(); + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); // Verify that file uri still exists even though file group is stale assertThat(fileStorage.exists(fileUri)).isTrue(); - mobileDataDownload.maintenance().get(); + mobileDataDownload.maintenance().get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); // Verify that file uri gets removed, once maintenance runs - assertThat(fileStorage.exists(fileUri)).isFalse(); + if (flags.mddEnableGarbageCollection()) { + // cl/439051122 created a temporary FALSE override targeted to ASGA devices. This test only + // makes sense if the flag is true, but all_on testing doesn't respect diversion criteria in + // the launch. So we skip it for now. + // TODO(b/226551373): remove this once AsgaDisableMddLibGcLaunch is turned down + assertThat(fileStorage.exists(fileUri)).isFalse(); + } } - // TODO: Improve this helper by getting rid of the need to new arrays when invoking - // and unnamed params. Something along this line: - // createDataFileGroup(name,package).addFile(..).addFile()... - // A helper function to create a DataFilegroup. - public static DataFileGroup createDataFileGroup( - String groupName, - String ownerPackage, - String[] fileId, - int[] byteSize, - String[] checksum, - ChecksumType[] checksumType, - String[] url, - DeviceNetworkPolicy deviceNetworkPolicy) { - if (fileId.length != byteSize.length - || fileId.length != checksum.length - || fileId.length != url.length - || checksumType.length != fileId.length) { - throw new IllegalArgumentException(); - } + @Test + public void handleTask_duplicateInvocations_logsDownloadCompleteOnce() throws Exception { + // Override the feature flag to log at 100%. + flags.mddDefaultSampleInterval = Optional.of(1); - DataFileGroup.Builder dataFileGroupBuilder = - DataFileGroup.newBuilder() - .setGroupName(groupName) - .setOwnerPackage(ownerPackage) - .setDownloadConditions( - DownloadConditions.newBuilder().setDeviceNetworkPolicy(deviceNetworkPolicy)); - - for (int i = 0; i < fileId.length; ++i) { - DataFile file = - DataFile.newBuilder() - .setFileId(fileId[i]) - .setByteSize(byteSize[i]) - .setChecksum(checksum[i]) - .setChecksumType(checksumType[i]) - .setUrlToDownload(url[i]) - .build(); - dataFileGroupBuilder.addFile(file); - } + TestFileDownloader testFileDownloader = + new TestFileDownloader( + TEST_DATA_RELATIVE_PATH, + fileStorage, + MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR)); + BlockingFileDownloader blockingFileDownloader = + new BlockingFileDownloader(DOWNLOAD_EXECUTOR, testFileDownloader); - return dataFileGroupBuilder.build(); - } + Supplier<FileDownloader> fakeFileDownloaderSupplier = () -> blockingFileDownloader; - private MobileDataDownload getMobileDataDownload( - Supplier<FileDownloader> fileDownloaderSupplier, FileGroupPopulator fileGroupPopulator) { - return getMobileDataDownloadBuilder(fileDownloaderSupplier, fileGroupPopulator).build(); + mobileDataDownload = + builderForTest().setFileDownloaderSupplier(fakeFileDownloaderSupplier).build(); + + // Use test populator to add the group as pending. + TestFileGroupPopulator populator = new TestFileGroupPopulator(context); + populator.refreshFileGroups(mobileDataDownload).get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); + + // Call handle task in non-blocking way and use blocking file downloader to let handleTask1 wait + // at the download stage + ListenableFuture<Void> handleTask1Future = + mobileDataDownload.handleTask(TaskScheduler.WIFI_CHARGING_PERIODIC_TASK); + + blockingFileDownloader.waitForDownloadStarted(); + + ListenableFuture<Void> handleTask2Future = + mobileDataDownload.handleTask(TaskScheduler.CELLULAR_CHARGING_PERIODIC_TASK); + + // Trigger a complete so the download "completes" after both tasks have been started. + blockingFileDownloader.finishDownloading(); + + // Wait for both futures to complete so we can make assertions about the events logged + handleTask2Future.get(MAX_HANDLE_TASK_WAIT_TIME_SECS, SECONDS); + handleTask1Future.get(MAX_HANDLE_TASK_WAIT_TIME_SECS, SECONDS); + + // Check that group is downloaded. + ClientFileGroup unused = getAndVerifyClientFileGroup(FILE_GROUP_NAME, 1); + + if (controlExecutorType.equals(ExecutorType.SINGLE_THREADED)) { + // Single-threaded executor step requires some time to allow logging to finish. + // TODO: Investigate whether TestingTaskBarrier can be used here to wait for + // executor become idle. + Thread.sleep(500); + } + + // Check that logger only logged 1 download complete event + ArgumentCaptor<MddLogData> logDataCompleteCaptor = ArgumentCaptor.forClass(MddLogData.class); + // 1007 is the tag number for MddClientEvent.Code.EVENT_CODE_UNSPECIFIED. + verify(mockLogger, times(1)).log(logDataCompleteCaptor.capture(), /* eventCode= */ eq(1007)); } - private MobileDataDownloadBuilder getMobileDataDownloadBuilder( - Supplier<FileDownloader> fileDownloaderSupplier, FileGroupPopulator fileGroupPopulator) { + private MobileDataDownloadBuilder builderForTest() { return MobileDataDownloadBuilder.newBuilder() .setContext(context) - .setControlExecutor(CONTROL_EXECUTOR) - .setFileDownloaderSupplier(fileDownloaderSupplier) - .addFileGroupPopulator(fileGroupPopulator) + .setControlExecutor(controlExecutor) .setTaskScheduler(Optional.of(mockTaskScheduler)) .setLoggerOptional(Optional.of(mockLogger)) .setDeltaDecoderOptional(Optional.absent()) @@ -845,12 +1034,8 @@ public class MobileDataDownloadIntegrationTest { } /** Creates MDD object and triggers handleTask to refresh and download file groups. */ - private MobileDataDownload getMobileDataDownloadAfterDownload( - Supplier<FileDownloader> fileDownloaderSupplier, FileGroupPopulator fileGroupPopulator) + private void waitForHandleTask() throws InterruptedException, ExecutionException, TimeoutException { - MobileDataDownload mobileDataDownload = - getMobileDataDownload(fileDownloaderSupplier, fileGroupPopulator); - // This will trigger refreshing of FileGroupPopulators and downloading. mobileDataDownload .handleTask(TaskScheduler.CELLULAR_CHARGING_PERIODIC_TASK) @@ -861,16 +1046,14 @@ public class MobileDataDownloadIntegrationTest { for (String line : debugString.split("\n", -1)) { Log.i(TAG, line); } - return mobileDataDownload; } - private static ClientFileGroup getAndVerifyClientFileGroup( - MobileDataDownload mobileDataDownload, String fileGroupName, int fileCount) - throws ExecutionException, InterruptedException { + private ClientFileGroup getAndVerifyClientFileGroup(String fileGroupName, int fileCount) + throws ExecutionException, TimeoutException, InterruptedException { ClientFileGroup clientFileGroup = mobileDataDownload .getFileGroup(GetFileGroupRequest.newBuilder().setGroupName(fileGroupName).build()) - .get(); + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); assertThat(clientFileGroup).isNotNull(); assertThat(clientFileGroup.getGroupName()).isEqualTo(fileGroupName); assertThat(clientFileGroup.getFileCount()).isEqualTo(fileCount); diff --git a/javatests/com/google/android/libraries/mobiledatadownload/MobileDataDownloadIsolatedStructuresIntegrationTest.java b/javatests/com/google/android/libraries/mobiledatadownload/MobileDataDownloadIsolatedStructuresIntegrationTest.java new file mode 100644 index 0000000..cbfe315 --- /dev/null +++ b/javatests/com/google/android/libraries/mobiledatadownload/MobileDataDownloadIsolatedStructuresIntegrationTest.java @@ -0,0 +1,227 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload; + +import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_CHECKSUM; +import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_ID; +import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_SIZE; +import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_URL; +import static com.google.android.libraries.mobiledatadownload.testing.MddTestDependencies.ExecutorType; +import static com.google.common.truth.Truth.assertThat; +import static java.util.concurrent.TimeUnit.SECONDS; + +import android.accounts.Account; +import android.content.Context; +import androidx.test.core.app.ApplicationProvider; +import com.google.android.libraries.mobiledatadownload.account.AccountUtil; +import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; +import com.google.android.libraries.mobiledatadownload.file.backends.AndroidFileBackend; +import com.google.android.libraries.mobiledatadownload.file.backends.JavaFileBackend; +import com.google.android.libraries.mobiledatadownload.monitor.NetworkUsageMonitor; +import com.google.android.libraries.mobiledatadownload.testing.FakeTimeSource; +import com.google.android.libraries.mobiledatadownload.testing.MddTestDependencies; +import com.google.android.libraries.mobiledatadownload.testing.TestFileDownloader; +import com.google.android.libraries.mobiledatadownload.testing.TestFlags; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.ListeningScheduledExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.mobiledatadownload.ClientConfigProto.ClientFileGroup; +import com.google.mobiledatadownload.DownloadConfigProto.DataFile; +import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; +import com.google.mobiledatadownload.DownloadConfigProto.DownloadConditions; +import com.google.mobiledatadownload.DownloadConfigProto.DownloadConditions.DeviceNetworkPolicy; +import com.google.testing.junit.testparameterinjector.TestParameter; +import com.google.testing.junit.testparameterinjector.TestParameterInjector; +import java.util.concurrent.Executors; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** + * Integration Tests that relate to interactions with MDD's Isolated Structures feature + * + * <p>Tests should be included here if they test MDD's behavior regarding reading/writing isolated + * structure groups. + */ +@RunWith(TestParameterInjector.class) +public class MobileDataDownloadIsolatedStructuresIntegrationTest { + + private static final String TAG = "MDDIsolatedStructuresIntegrationTest"; + private static final int MAX_MDD_API_WAIT_TIME_SECS = 5; + + private static final String GROUP_NAME_1 = "test-group-1"; + private static final String GROUP_NAME_2 = "test-group-2"; + + private static final String VARIANT_1 = "test-variant-1"; + private static final String VARIANT_2 = "test-variant-2"; + + private static final Account ACCOUNT_1 = AccountUtil.create("account-name-1", "account-type"); + private static final Account ACCOUNT_2 = AccountUtil.create("account-name-2", "account-type"); + + private static final String TEST_DATA_RELATIVE_PATH = + "third_party/java_src/android_libs/mobiledatadownload/javatests/com/google/android/libraries/mobiledatadownload/testdata/"; + + private static final ListeningScheduledExecutorService DOWNLOAD_EXECUTOR = + MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(2)); + + private static final Context context = ApplicationProvider.getApplicationContext(); + + private final NetworkUsageMonitor networkUsageMonitor = + new NetworkUsageMonitor(context, new FakeTimeSource()); + + private final SynchronousFileStorage fileStorage = + new SynchronousFileStorage( + ImmutableList.of(AndroidFileBackend.builder(context).build(), new JavaFileBackend()), + ImmutableList.of(), + ImmutableList.of(networkUsageMonitor)); + + private final TestFlags flags = new TestFlags(); + + private ListeningExecutorService controlExecutor; + + @Mock private Logger mockLogger; + @Mock private TaskScheduler mockTaskScheduler; + + @Rule(order = 1) + public final MockitoRule mocks = MockitoJUnit.rule(); + + @TestParameter ExecutorType controlExecutorType; + + @Before + public void setUp() throws Exception { + + controlExecutor = controlExecutorType.executor(); + } + + @Test + public void addFileGroup_whenImmediatelyComplete_createsCorrectIsolatedRoot( + @TestParameter boolean sameGroupName, + @TestParameter boolean sameAccount, + @TestParameter boolean sameVariantId) + throws Exception { + Optional<String> instanceId = Optional.of(MddTestDependencies.randomInstanceId()); + + String groupName1 = GROUP_NAME_1; + String variantId1 = VARIANT_1; + Account account1 = ACCOUNT_1; + + // Define group2 properties based on test parameters + String groupName2 = sameGroupName ? GROUP_NAME_1 : GROUP_NAME_2; + String variantId2 = sameVariantId ? VARIANT_1 : VARIANT_2; + Account account2 = sameAccount ? ACCOUNT_1 : ACCOUNT_2; + + DataFileGroup symlinkGroup1 = buildSymlinkGroup(groupName1, variantId1); + DataFileGroup symlinkGroup2 = buildSymlinkGroup(groupName2, variantId2); + + MobileDataDownload mobileDataDownload = + builderForTest() + .setInstanceIdOptional(instanceId) + .setFileDownloaderSupplier( + () -> + new TestFileDownloader( + TEST_DATA_RELATIVE_PATH, + fileStorage, + MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR))) + .build(); + + // Add group1 and download it + mobileDataDownload + .addFileGroup( + AddFileGroupRequest.newBuilder() + .setDataFileGroup(symlinkGroup1) + .setVariantIdOptional(Optional.of(variantId1)) + .setAccountOptional(Optional.of(account1)) + .build()) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); + ClientFileGroup downloadedSymlinkGroup1 = + mobileDataDownload + .downloadFileGroup( + DownloadFileGroupRequest.newBuilder() + .setGroupName(groupName1) + .setVariantIdOptional(Optional.of(variantId1)) + .setAccountOptional(Optional.of(account1)) + .build()) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); + + // Add group2 and get it since it should be immediately downloaded. + mobileDataDownload + .addFileGroup( + AddFileGroupRequest.newBuilder() + .setDataFileGroup(symlinkGroup2) + .setVariantIdOptional(Optional.of(variantId2)) + .setAccountOptional(Optional.of(account2)) + .build()) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); + ClientFileGroup downloadedSymlinkGroup2 = + mobileDataDownload + .getFileGroup( + GetFileGroupRequest.newBuilder() + .setGroupName(groupName2) + .setVariantIdOptional(Optional.of(variantId2)) + .setAccountOptional(Optional.of(account2)) + .build()) + .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); + + String isolatedFileUriGroup1 = downloadedSymlinkGroup1.getFile(0).getFileUri(); + String isolatedFileUriGroup2 = downloadedSymlinkGroup2.getFile(0).getFileUri(); + assertThat(isolatedFileUriGroup1).contains(groupName1 + "_"); + assertThat(isolatedFileUriGroup2).contains(groupName2 + "_"); + + // assert that uris are the same if all test parameters are true and different if otherwise. + assertThat(isolatedFileUriGroup1.equalsIgnoreCase(isolatedFileUriGroup2)) + .isEqualTo(sameGroupName && sameVariantId && sameAccount); + } + + private static DataFileGroup buildSymlinkGroup(String groupName, String variantId) { + return DataFileGroup.newBuilder() + .setOwnerPackage(context.getPackageName()) + .setGroupName(groupName) + .setDownloadConditions( + DownloadConditions.newBuilder() + .setDeviceNetworkPolicy(DeviceNetworkPolicy.DOWNLOAD_ON_ANY_NETWORK) + .build()) + .addFile( + DataFile.newBuilder() + .setFileId(FILE_ID) + .setByteSize(FILE_SIZE) + .setChecksum(FILE_CHECKSUM) + .setUrlToDownload(FILE_URL) + .setRelativeFilePath("my-file.tmp") + .build()) + .setPreserveFilenamesAndIsolateFiles(true) + .setVariantId(variantId) + .setBuildId(9999) + .build(); + } + + private MobileDataDownloadBuilder builderForTest() { + return MobileDataDownloadBuilder.newBuilder() + .setContext(context) + .setControlExecutor(controlExecutor) + .setTaskScheduler(Optional.of(mockTaskScheduler)) + .setLoggerOptional(Optional.of(mockLogger)) + .setDeltaDecoderOptional(Optional.absent()) + .setFileStorage(fileStorage) + .setNetworkUsageMonitor(networkUsageMonitor) + .setFlagsOptional(Optional.of(flags)); + } +} diff --git a/javatests/com/google/android/libraries/mobiledatadownload/MobileDataDownloadTest.java b/javatests/com/google/android/libraries/mobiledatadownload/MobileDataDownloadTest.java index b98cd36..f695a20 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/MobileDataDownloadTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/MobileDataDownloadTest.java @@ -27,6 +27,7 @@ import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; @@ -36,7 +37,6 @@ import static org.mockito.Mockito.when; import android.accounts.Account; import android.content.Context; import android.net.Uri; -import android.util.Pair; import androidx.test.core.app.ApplicationProvider; import com.google.android.libraries.mobiledatadownload.DownloadException.DownloadResultCode; import com.google.android.libraries.mobiledatadownload.TaskScheduler.ConstraintOverrides; @@ -46,10 +46,14 @@ import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStora import com.google.android.libraries.mobiledatadownload.file.backends.AndroidFileBackend; import com.google.android.libraries.mobiledatadownload.file.openers.WriteStreamOpener; import com.google.android.libraries.mobiledatadownload.internal.MobileDataDownloadManager; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup; import com.google.android.libraries.mobiledatadownload.internal.logging.EventLogger; +import com.google.android.libraries.mobiledatadownload.internal.logging.testing.FakeEventLogger; import com.google.android.libraries.mobiledatadownload.internal.util.ProtoConversionUtil; import com.google.android.libraries.mobiledatadownload.lite.Downloader; import com.google.android.libraries.mobiledatadownload.monitor.DownloadProgressMonitor; +import com.google.android.libraries.mobiledatadownload.testing.FakeTimeSource; +import com.google.android.libraries.mobiledatadownload.testing.TestFlags; import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -57,6 +61,7 @@ import com.google.common.labs.concurrent.LabsFutures; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.mobiledatadownload.ClientConfigProto.ClientFile; import com.google.mobiledatadownload.ClientConfigProto.ClientFileGroup; @@ -65,6 +70,7 @@ import com.google.mobiledatadownload.DownloadConfigProto.DataFile; import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; import com.google.mobiledatadownload.DownloadConfigProto.DownloadConditions; import com.google.mobiledatadownload.DownloadConfigProto.DownloadConditions.DeviceNetworkPolicy; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; import com.google.mobiledatadownload.internal.MetadataProto; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; @@ -77,12 +83,12 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executor; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -97,9 +103,7 @@ import org.robolectric.RobolectricTestRunner; /** Tests for {@link com.google.android.libraries.mobiledatadownload.MobileDataDownload}. */ @RunWith(RobolectricTestRunner.class) public class MobileDataDownloadTest { - // Note: Control Executor must not be a single thread executor. - private static final Executor EXECUTOR = Executors.newCachedThreadPool(); - private static final long LATCH_WAIT_TIME_MS = 1000L; + private static final Context context = ApplicationProvider.getApplicationContext(); private static final String FILE_GROUP_NAME_1 = "test-group-1"; private static final String FILE_GROUP_NAME_2 = "test-group-2"; @@ -113,6 +117,28 @@ public class MobileDataDownloadTest { private static final String FILE_URL_2 = "https://www.gstatic.com/suggest-dev/odws1_empty.jar"; private static final int FILE_SIZE_2 = 554; + private static final DataFileGroup FILE_GROUP_1 = + createDataFileGroup( + FILE_GROUP_NAME_1, + context.getPackageName(), + /* versionNumber= */ 1, + new String[] {FILE_ID_1}, + new int[] {FILE_SIZE_1}, + new String[] {FILE_CHECKSUM_1}, + new String[] {FILE_URL_1}, + DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI); + + private static final DataFileGroupInternal FILE_GROUP_INTERNAL_1 = + createDataFileGroupInternal( + FILE_GROUP_NAME_1, + context.getPackageName(), + /* versionNumber= */ 5, + new String[] {FILE_ID_1}, + new int[] {FILE_SIZE_1}, + new String[] {FILE_CHECKSUM_1}, + new String[] {FILE_URL_1}, + DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI); + private final Uri onDeviceUri1 = Uri.parse( "android://com.google.android.libraries.mobiledatadownload/files/datadownload/shared/public/file_1"); @@ -132,9 +158,10 @@ public class MobileDataDownloadTest { "android://com.google.android.libraries.mobiledatadownload/files/datadownload/shared/public/dir/sub/file"); private final String onDeviceDirFile3Content = "Test file 3 in sub-dir."; - private final Flags flags = new Flags() {}; - private Context context; + private final TestFlags flags = new TestFlags(); private SynchronousFileStorage fileStorage; + private FakeTimeSource timeSource; + private FakeEventLogger fakeEventLogger; @Mock EventLogger mockEventLogger; @Mock MobileDataDownloadManager mockMobileDataDownloadManager; @@ -146,19 +173,42 @@ public class MobileDataDownloadTest { @Captor ArgumentCaptor<GroupKey> groupKeyCaptor; @Captor ArgumentCaptor<List<GroupKey>> groupKeysCaptor; + // Note: Executor must not be a single thread executor. + ListeningExecutorService controlExecutor = + MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); + @Rule public final MockitoRule mocks = MockitoJUnit.rule(); @Before public void setUp() throws IOException { - context = ApplicationProvider.getApplicationContext(); fileStorage = new SynchronousFileStorage( ImmutableList.of(AndroidFileBackend.builder(context).build()) /*backends*/); createFile(onDeviceUri1, "test"); - fileStorage.createDirectory(onDeviceDirUri); + if (!fileStorage.exists(onDeviceDirUri)) { + fileStorage.createDirectory(onDeviceDirUri); + } createFile(onDeviceDirFileUri1, onDeviceDirFile1Content); createFile(onDeviceDirFileUri2, onDeviceDirFile2Content); createFile(onDeviceDirFileUri3, onDeviceDirFile3Content); + timeSource = new FakeTimeSource(); + fakeEventLogger = new FakeEventLogger(); + } + + @After + public void tearDown() throws Exception { + if (fileStorage.exists(onDeviceUri1)) { + fileStorage.deleteFile(onDeviceUri1); + } + if (fileStorage.exists(onDeviceDirFileUri1)) { + fileStorage.deleteFile(onDeviceDirFileUri1); + } + if (fileStorage.exists(onDeviceDirFileUri2)) { + fileStorage.deleteFile(onDeviceDirFileUri2); + } + if (fileStorage.exists(onDeviceDirFileUri3)) { + fileStorage.deleteFile(onDeviceDirFileUri3); + } } private void createFile(Uri uri, String content) throws IOException { @@ -167,6 +217,8 @@ public class MobileDataDownloadTest { } } + private void expectErrorLogMessage(String message) {} + @Test public void buildGetFileGroupsByFilterRequest() throws Exception { Account account = AccountUtil.create("account-name", "account-type"); @@ -208,23 +260,13 @@ public class MobileDataDownloadTest { when(mockMobileDataDownloadManager.addGroupForDownloadInternal( any(GroupKey.class), any(DataFileGroupInternal.class), any())) .thenReturn(Futures.immediateFuture(true)); - DataFileGroup dataFileGroup = - createDataFileGroup( - FILE_GROUP_NAME_1, - context.getPackageName(), - 1 /* versionNumber */, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI); MobileDataDownload mobileDataDownload = new MobileDataDownloadImpl( context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -232,12 +274,13 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); assertThat( mobileDataDownload .addFileGroup( - AddFileGroupRequest.newBuilder().setDataFileGroup(dataFileGroup).build()) + AddFileGroupRequest.newBuilder().setDataFileGroup(FILE_GROUP_1).build()) .get()) .isTrue(); } @@ -247,23 +290,13 @@ public class MobileDataDownloadTest { when(mockMobileDataDownloadManager.addGroupForDownloadInternal( any(GroupKey.class), any(DataFileGroupInternal.class), any())) .thenReturn(Futures.immediateFuture(false)); - DataFileGroup dataFileGroup = - createDataFileGroup( - FILE_GROUP_NAME_1, - context.getPackageName(), - 1 /* versionNumber */, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI); MobileDataDownload mobileDataDownload = new MobileDataDownloadImpl( context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -271,12 +304,13 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); assertThat( mobileDataDownload .addFileGroup( - AddFileGroupRequest.newBuilder().setDataFileGroup(dataFileGroup).build()) + AddFileGroupRequest.newBuilder().setDataFileGroup(FILE_GROUP_1).build()) .get()) .isFalse(); } @@ -304,7 +338,7 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, null /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -312,8 +346,12 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); + expectErrorLogMessage( + "MobileDataDownload: Added group = 'test-group-1' with wrong owner package:" + + " 'com.google.android.libraries.mobiledatadownload' v.s. 'PACKAGE_NAME' "); assertThat( mobileDataDownload .addFileGroup( @@ -329,23 +367,12 @@ public class MobileDataDownloadTest { groupKeyCaptor.capture(), any(DataFileGroupInternal.class), any())) .thenReturn(Futures.immediateFuture(true)); - DataFileGroup dataFileGroup = - createDataFileGroup( - FILE_GROUP_NAME_1, - context.getPackageName(), - 1 /* versionNumber */, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI); - MobileDataDownload mobileDataDownload = new MobileDataDownloadImpl( context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -353,12 +380,13 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); Account account = AccountUtil.create("account-name", "account-type"); AddFileGroupRequest addFileGroupRequest = AddFileGroupRequest.newBuilder() - .setDataFileGroup(dataFileGroup) + .setDataFileGroup(FILE_GROUP_1) .setAccountOptional(Optional.of(account)) .build(); @@ -373,7 +401,7 @@ public class MobileDataDownloadTest { assertThat(groupKeyCaptor.getValue()).isEqualTo(groupKey); verify(mockMobileDataDownloadManager) .addGroupForDownloadInternal( - eq(groupKey), eq(ProtoConversionUtil.convert(dataFileGroup)), any()); + eq(groupKey), eq(ProtoConversionUtil.convert(FILE_GROUP_1)), any()); } @Test @@ -383,23 +411,12 @@ public class MobileDataDownloadTest { groupKeyCaptor.capture(), any(DataFileGroupInternal.class), any())) .thenReturn(Futures.immediateFuture(false)); - DataFileGroup dataFileGroup = - createDataFileGroup( - FILE_GROUP_NAME_1, - context.getPackageName(), - 1 /* versionNumber */, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI); - MobileDataDownload mobileDataDownload = new MobileDataDownloadImpl( context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -407,12 +424,13 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); Account account = AccountUtil.create("account-name", "account-type"); AddFileGroupRequest addFileGroupRequest = AddFileGroupRequest.newBuilder() - .setDataFileGroup(dataFileGroup) + .setDataFileGroup(FILE_GROUP_1) .setAccountOptional(Optional.of(account)) .build(); @@ -427,7 +445,7 @@ public class MobileDataDownloadTest { assertThat(groupKeyCaptor.getValue()).isEqualTo(groupKey); verify(mockMobileDataDownloadManager) .addGroupForDownloadInternal( - eq(groupKey), eq(ProtoConversionUtil.convert(dataFileGroup)), any()); + eq(groupKey), eq(ProtoConversionUtil.convert(FILE_GROUP_1)), any()); } @Test @@ -453,7 +471,7 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -461,7 +479,8 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); AddFileGroupRequest addFileGroupRequest = AddFileGroupRequest.newBuilder().setDataFileGroup(dataFileGroup).build(); @@ -480,62 +499,6 @@ public class MobileDataDownloadTest { } @Test - public void addFileGroupWithFileGroupKey_withVariant() throws Exception { - ArgumentCaptor<GroupKey> groupKeyCaptor = ArgumentCaptor.forClass(GroupKey.class); - when(mockMobileDataDownloadManager.addGroupForDownloadInternal( - groupKeyCaptor.capture(), any(), any())) - .thenReturn(Futures.immediateFuture(true)); - - DataFileGroup dataFileGroupWithVariant = - createDataFileGroup( - FILE_GROUP_NAME_1, - context.getPackageName(), - 1 /* versionNumber */, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI) - .toBuilder() - .setVariantId("en") - .build(); - - MobileDataDownload mobileDataDownload = - new MobileDataDownloadImpl( - context, - mockEventLogger, - mockMobileDataDownloadManager, - EXECUTOR, - ImmutableList.of() /* fileGroupPopulatorList */, - Optional.of(mockTaskScheduler), - fileStorage, - Optional.absent() /* downloadMonitorOptional */, - Optional.of(this.getClass()), // don't need to use the real foreground download service. - flags, - singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); - - AddFileGroupRequest addFileGroupRequest = - AddFileGroupRequest.newBuilder() - .setDataFileGroup(dataFileGroupWithVariant) - .setVariantIdOptional(Optional.of("en")) - .build(); - - assertThat(mobileDataDownload.addFileGroup(addFileGroupRequest).get()).isTrue(); - - GroupKey groupKey = - GroupKey.newBuilder() - .setGroupName(FILE_GROUP_NAME_1) - .setOwnerPackage(context.getPackageName()) - .setVariantId("en") - .build(); - assertThat(groupKeyCaptor.getValue()).isEqualTo(groupKey); - verify(mockMobileDataDownloadManager) - .addGroupForDownloadInternal( - eq(groupKey), eq(ProtoConversionUtil.convert(dataFileGroupWithVariant)), any()); - } - - @Test public void removeFileGroup_onSuccess_returnsTrue() throws Exception { ArgumentCaptor<GroupKey> groupKeyCaptor = ArgumentCaptor.forClass(GroupKey.class); when(mockMobileDataDownloadManager.removeFileGroup(groupKeyCaptor.capture(), eq(false))) @@ -546,7 +509,7 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -554,7 +517,8 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); RemoveFileGroupRequest removeFileGroupRequest = RemoveFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME_1).build(); @@ -581,7 +545,7 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -589,7 +553,8 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); RemoveFileGroupRequest removeFileGroupRequest = RemoveFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME_1).build(); @@ -620,7 +585,7 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -628,7 +593,8 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); Account account = AccountUtil.create("account-name", "account-type"); RemoveFileGroupRequest removeFileGroupRequest = @@ -660,7 +626,7 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -668,7 +634,8 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); RemoveFileGroupRequest removeFileGroupRequest = RemoveFileGroupRequest.newBuilder() @@ -690,30 +657,20 @@ public class MobileDataDownloadTest { @Test public void getFileGroup() throws Exception { DataFileGroupInternal dataFileGroup = - createDataFileGroupInternal( - FILE_GROUP_NAME_1, - context.getPackageName(), - 5 /* versionNumber */, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI) - .toBuilder() - .setBuildId(10) - .setVariantId("test-variant") - .build(); + FILE_GROUP_INTERNAL_1.toBuilder().setBuildId(10).setVariantId("test-variant").build(); when(mockMobileDataDownloadManager.getFileGroup(any(GroupKey.class), eq(true))) .thenReturn(Futures.immediateFuture(dataFileGroup)); - when(mockMobileDataDownloadManager.getDataFileUri(dataFileGroup.getFile(0), dataFileGroup)) - .thenReturn(Futures.immediateFuture(onDeviceUri1)); + when(mockMobileDataDownloadManager.getDataFileUris( + dataFileGroup, /* verifyIsolatedStructure= */ true)) + .thenReturn( + Futures.immediateFuture(ImmutableMap.of(dataFileGroup.getFile(0), onDeviceUri1))); MobileDataDownload mobileDataDownload = new MobileDataDownloadImpl( context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -721,7 +678,8 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); ClientFileGroup clientFileGroup = mobileDataDownload @@ -740,27 +698,20 @@ public class MobileDataDownloadTest { @Test public void getFileGroup_withDirectory() throws Exception { - DataFileGroupInternal dataFileGroup = - createDataFileGroupInternal( - FILE_GROUP_NAME_1, - context.getPackageName(), - 5 /* versionNumber */, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI); when(mockMobileDataDownloadManager.getFileGroup(any(GroupKey.class), eq(true))) - .thenReturn(Futures.immediateFuture(dataFileGroup)); - when(mockMobileDataDownloadManager.getDataFileUri(dataFileGroup.getFile(0), dataFileGroup)) - .thenReturn(Futures.immediateFuture(onDeviceDirUri)); + .thenReturn(Futures.immediateFuture(FILE_GROUP_INTERNAL_1)); + when(mockMobileDataDownloadManager.getDataFileUris( + FILE_GROUP_INTERNAL_1, /* verifyIsolatedStructure= */ true)) + .thenReturn( + Futures.immediateFuture( + ImmutableMap.of(FILE_GROUP_INTERNAL_1.getFile(0), onDeviceDirUri))); MobileDataDownload mobileDataDownload = new MobileDataDownloadImpl( context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -768,7 +719,8 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); ClientFileGroup clientFileGroup = mobileDataDownload @@ -806,27 +758,20 @@ public class MobileDataDownloadTest { @Test public void getFileGroup_withDirectory_withTraverseDisabled() throws Exception { - DataFileGroupInternal dataFileGroup = - createDataFileGroupInternal( - FILE_GROUP_NAME_1, - context.getPackageName(), - 5 /* versionNumber */, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI); when(mockMobileDataDownloadManager.getFileGroup(any(GroupKey.class), eq(true))) - .thenReturn(Futures.immediateFuture(dataFileGroup)); - when(mockMobileDataDownloadManager.getDataFileUri(dataFileGroup.getFile(0), dataFileGroup)) - .thenReturn(Futures.immediateFuture(onDeviceDirUri)); + .thenReturn(Futures.immediateFuture(FILE_GROUP_INTERNAL_1)); + when(mockMobileDataDownloadManager.getDataFileUris( + FILE_GROUP_INTERNAL_1, /* verifyIsolatedStructure= */ true)) + .thenReturn( + Futures.immediateFuture( + ImmutableMap.of(FILE_GROUP_INTERNAL_1.getFile(0), onDeviceDirUri))); MobileDataDownload mobileDataDownload = new MobileDataDownloadImpl( context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -834,7 +779,8 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); ClientFileGroup clientFileGroup = mobileDataDownload @@ -864,34 +810,13 @@ public class MobileDataDownloadTest { @Test public void removeFileGroupsByFilter_withAccountSpecified_removesMatchingAccountGroups() throws Exception { - List<Pair<GroupKey, DataFileGroupInternal>> keyToGroupList = new ArrayList<>(); + List<GroupKeyAndGroup> keyToGroupList = new ArrayList<>(); Account account1 = AccountUtil.create("account-name", "account-type"); Account account2 = AccountUtil.create("account-name2", "account-type"); - DataFileGroupInternal downloadedFileGroup = - createDataFileGroupInternal( - FILE_GROUP_NAME_1, - context.getPackageName(), - /* versionNumber = */ 5, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI) - .toBuilder() - .build(); + DataFileGroupInternal downloadedFileGroup = FILE_GROUP_INTERNAL_1.toBuilder().build(); DataFileGroupInternal pendingFileGroup = - createDataFileGroupInternal( - FILE_GROUP_NAME_1, - context.getPackageName(), - /* versionNumber = */ 6, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI) - .toBuilder() - .build(); + FILE_GROUP_INTERNAL_1.toBuilder().setFileGroupVersionNumber(6).build(); GroupKey account1GroupKey = GroupKey.newBuilder() @@ -919,12 +844,12 @@ public class MobileDataDownloadTest { GroupKey downloadedGroupKey = noAccountGroupKey.toBuilder().setDownloaded(true).build(); GroupKey pendingGroupKey = noAccountGroupKey.toBuilder().setDownloaded(false).build(); - keyToGroupList.add(Pair.create(downloadedGroupKey, downloadedFileGroup)); - keyToGroupList.add(Pair.create(downloadedAccount1GroupKey, downloadedFileGroup)); - keyToGroupList.add(Pair.create(downloadedAccount2GroupKey, downloadedFileGroup)); - keyToGroupList.add(Pair.create(pendingGroupKey, pendingFileGroup)); - keyToGroupList.add(Pair.create(pendingAccount1GroupKey, pendingFileGroup)); - keyToGroupList.add(Pair.create(pendingAccount2GroupKey, pendingFileGroup)); + keyToGroupList.add(GroupKeyAndGroup.create(downloadedGroupKey, downloadedFileGroup)); + keyToGroupList.add(GroupKeyAndGroup.create(downloadedAccount1GroupKey, downloadedFileGroup)); + keyToGroupList.add(GroupKeyAndGroup.create(downloadedAccount2GroupKey, downloadedFileGroup)); + keyToGroupList.add(GroupKeyAndGroup.create(pendingGroupKey, pendingFileGroup)); + keyToGroupList.add(GroupKeyAndGroup.create(pendingAccount1GroupKey, pendingFileGroup)); + keyToGroupList.add(GroupKeyAndGroup.create(pendingAccount2GroupKey, pendingFileGroup)); when(mockMobileDataDownloadManager.getAllFreshGroups()) .thenReturn(Futures.immediateFuture(keyToGroupList)); @@ -936,15 +861,16 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, - /* fileGroupPopulatorList = */ ImmutableList.of(), + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), Optional.of(mockTaskScheduler), fileStorage, - /* downloadMonitorOptional = */ Optional.absent(), - /* foregroundDownloadServiceClassOptional = */ Optional.absent(), + /* downloadMonitorOptional= */ Optional.absent(), + /* foregroundDownloadServiceClassOptional= */ Optional.absent(), flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); // Setup request that matches all fresh groups, but also include account to make sure only // account associated file groups are removed @@ -964,19 +890,10 @@ public class MobileDataDownloadTest { @Test public void getFileGroup_nullFileUri() throws Exception { - DataFileGroupInternal dataFileGroup = - createDataFileGroupInternal( - FILE_GROUP_NAME_1, - context.getPackageName(), - 5 /* versionNumber */, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI); when(mockMobileDataDownloadManager.getFileGroup(any(GroupKey.class), eq(true))) - .thenReturn(Futures.immediateFuture(dataFileGroup)); - when(mockMobileDataDownloadManager.getDataFileUri(dataFileGroup.getFile(0), dataFileGroup)) + .thenReturn(Futures.immediateFuture(FILE_GROUP_INTERNAL_1)); + when(mockMobileDataDownloadManager.getDataFileUris( + FILE_GROUP_INTERNAL_1, /* verifyIsolatedStructure= */ true)) .thenReturn( Futures.immediateFailedFuture( DownloadException.builder() @@ -989,7 +906,7 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -997,7 +914,8 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); assertNull( mobileDataDownload @@ -1015,7 +933,7 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -1023,40 +941,32 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); assertNull( mobileDataDownload .getFileGroup(GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME_1).build()) .get()); - - verifyNoInteractions(mockEventLogger); } @Test public void getFileGroup_withAccount() throws Exception { - DataFileGroupInternal dataFileGroup = - createDataFileGroupInternal( - FILE_GROUP_NAME_1, - context.getPackageName(), - 5 /* versionNumber */, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI); ArgumentCaptor<GroupKey> groupKeyCaptor = ArgumentCaptor.forClass(GroupKey.class); when(mockMobileDataDownloadManager.getFileGroup(groupKeyCaptor.capture(), eq(true))) - .thenReturn(Futures.immediateFuture(dataFileGroup)); - when(mockMobileDataDownloadManager.getDataFileUri(dataFileGroup.getFile(0), dataFileGroup)) - .thenReturn(Futures.immediateFuture(onDeviceUri1)); + .thenReturn(Futures.immediateFuture(FILE_GROUP_INTERNAL_1)); + when(mockMobileDataDownloadManager.getDataFileUris( + FILE_GROUP_INTERNAL_1, /* verifyIsolatedStructure= */ true)) + .thenReturn( + Futures.immediateFuture( + ImmutableMap.of(FILE_GROUP_INTERNAL_1.getFile(0), onDeviceUri1))); MobileDataDownload mobileDataDownload = new MobileDataDownloadImpl( context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -1064,7 +974,8 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); Account account = AccountUtil.create("account-name", "account-type"); ClientFileGroup clientFileGroup = @@ -1097,31 +1008,22 @@ public class MobileDataDownloadTest { @Test public void getFileGroup_withVariantId() throws Exception { DataFileGroupInternal dataFileGroup = - createDataFileGroupInternal( - FILE_GROUP_NAME_1, - context.getPackageName(), - 5 /* versionNumber */, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI) - .toBuilder() - .setVariantId("en") - .build(); + FILE_GROUP_INTERNAL_1.toBuilder().setVariantId("en").build(); ArgumentCaptor<GroupKey> groupKeyCaptor = ArgumentCaptor.forClass(GroupKey.class); when(mockMobileDataDownloadManager.getFileGroup(groupKeyCaptor.capture(), eq(true))) .thenReturn(Futures.immediateFuture(dataFileGroup)); - when(mockMobileDataDownloadManager.getDataFileUri(dataFileGroup.getFile(0), dataFileGroup)) - .thenReturn(Futures.immediateFuture(onDeviceUri1)); + when(mockMobileDataDownloadManager.getDataFileUris( + dataFileGroup, /* verifyIsolatedStructure= */ true)) + .thenReturn( + Futures.immediateFuture(ImmutableMap.of(dataFileGroup.getFile(0), onDeviceUri1))); MobileDataDownload mobileDataDownload = new MobileDataDownloadImpl( context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -1129,7 +1031,8 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); ClientFileGroup clientFileGroup = mobileDataDownload @@ -1164,16 +1067,7 @@ public class MobileDataDownloadTest { .setValue(StringValue.of("TEST_PROPERTY").toByteString()) .build(); DataFileGroupInternal dataFileGroup = - createDataFileGroupInternal( - FILE_GROUP_NAME_1, - context.getPackageName(), - 5 /* versionNumber */, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI) - .toBuilder() + FILE_GROUP_INTERNAL_1.toBuilder() .setBuildId(1L) .setVariantId("testvariant") .setCustomProperty(customProperty) @@ -1181,15 +1075,17 @@ public class MobileDataDownloadTest { when(mockMobileDataDownloadManager.getFileGroup(any(GroupKey.class), eq(true))) .thenReturn(Futures.immediateFuture(dataFileGroup)); - when(mockMobileDataDownloadManager.getDataFileUri(dataFileGroup.getFile(0), dataFileGroup)) - .thenReturn(Futures.immediateFuture(onDeviceUri1)); + when(mockMobileDataDownloadManager.getDataFileUris( + dataFileGroup, /* verifyIsolatedStructure= */ true)) + .thenReturn( + Futures.immediateFuture(ImmutableMap.of(dataFileGroup.getFile(0), onDeviceUri1))); MobileDataDownload mobileDataDownload = new MobileDataDownloadImpl( context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -1197,7 +1093,8 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); ClientFileGroup clientFileGroup = mobileDataDownload @@ -1214,31 +1111,21 @@ public class MobileDataDownloadTest { @Test public void getFileGroup_includesLocale() throws Exception { DataFileGroupInternal dataFileGroup = - createDataFileGroupInternal( - FILE_GROUP_NAME_1, - context.getPackageName(), - 5 /* versionNumber */, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI) - .toBuilder() - .addLocale("en-US") - .addLocale("en-CA") - .build(); + FILE_GROUP_INTERNAL_1.toBuilder().addLocale("en-US").addLocale("en-CA").build(); when(mockMobileDataDownloadManager.getFileGroup(any(GroupKey.class), eq(true))) .thenReturn(Futures.immediateFuture(dataFileGroup)); - when(mockMobileDataDownloadManager.getDataFileUri(dataFileGroup.getFile(0), dataFileGroup)) - .thenReturn(Futures.immediateFuture(onDeviceUri1)); + when(mockMobileDataDownloadManager.getDataFileUris( + dataFileGroup, /* verifyIsolatedStructure= */ true)) + .thenReturn( + Futures.immediateFuture(ImmutableMap.of(dataFileGroup.getFile(0), onDeviceUri1))); MobileDataDownload mobileDataDownload = new MobileDataDownloadImpl( context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -1246,7 +1133,8 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); ClientFileGroup clientFileGroup = mobileDataDownload @@ -1265,30 +1153,21 @@ public class MobileDataDownloadTest { .setValue(StringValue.of("TEST_METADATA").toByteString()) .build(); DataFileGroupInternal dataFileGroup = - createDataFileGroupInternal( - FILE_GROUP_NAME_1, - context.getPackageName(), - 5 /* versionNumber */, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI) - .toBuilder() - .setCustomMetadata(customMetadata) - .build(); + FILE_GROUP_INTERNAL_1.toBuilder().setCustomMetadata(customMetadata).build(); when(mockMobileDataDownloadManager.getFileGroup(any(GroupKey.class), eq(true))) .thenReturn(Futures.immediateFuture(dataFileGroup)); - when(mockMobileDataDownloadManager.getDataFileUri(dataFileGroup.getFile(0), dataFileGroup)) - .thenReturn(Futures.immediateFuture(onDeviceUri1)); + when(mockMobileDataDownloadManager.getDataFileUris( + dataFileGroup, /* verifyIsolatedStructure= */ true)) + .thenReturn( + Futures.immediateFuture(ImmutableMap.of(dataFileGroup.getFile(0), onDeviceUri1))); MobileDataDownload mobileDataDownload = new MobileDataDownloadImpl( context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -1296,7 +1175,8 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); ClientFileGroup clientFileGroup = mobileDataDownload @@ -1333,17 +1213,22 @@ public class MobileDataDownloadTest { when(mockMobileDataDownloadManager.getFileGroup(any(GroupKey.class), eq(true))) .thenReturn(Futures.immediateFuture(dataFileGroup)); - when(mockMobileDataDownloadManager.getDataFileUri(dataFileGroup.getFile(0), dataFileGroup)) - .thenReturn(Futures.immediateFuture(onDeviceUri1)); - when(mockMobileDataDownloadManager.getDataFileUri(dataFileGroup.getFile(1), dataFileGroup)) - .thenReturn(Futures.immediateFuture(onDeviceUri1)); + when(mockMobileDataDownloadManager.getDataFileUris( + dataFileGroup, /* verifyIsolatedStructure= */ true)) + .thenReturn( + Futures.immediateFuture( + ImmutableMap.of( + dataFileGroup.getFile(0), + onDeviceUri1, + dataFileGroup.getFile(1), + onDeviceUri1))); MobileDataDownload mobileDataDownload = new MobileDataDownloadImpl( context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -1351,7 +1236,8 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); ClientFileGroup clientFileGroup = mobileDataDownload @@ -1376,19 +1262,456 @@ public class MobileDataDownloadTest { } @Test + public void getFileGroup_whenVerifyIsolatedStructureIsFalse_skipsStructureVerification() + throws Exception { + MetadataProto.DataFile isolatedStructureFile = + MetadataProto.DataFile.newBuilder() + .setFileId(FILE_ID_1) + .setChecksumType(MetadataProto.DataFile.ChecksumType.NONE) + .setUrlToDownload(FILE_URL_1) + .setRelativeFilePath("mycustom/file.txt") + .build(); + DataFileGroupInternal isolatedStructureGroup = + DataFileGroupInternal.newBuilder() + .setGroupName(FILE_GROUP_NAME_1) + .setOwnerPackage(context.getPackageName()) + .setPreserveFilenamesAndIsolateFiles(true) + .addFile(isolatedStructureFile) + .build(); + + when(mockMobileDataDownloadManager.getFileGroup(any(GroupKey.class), eq(true))) + .thenReturn(Futures.immediateFuture(isolatedStructureGroup)); + when(mockMobileDataDownloadManager.getDataFileUris( + isolatedStructureGroup, /* verifyIsolatedStructure= */ false)) + .thenReturn( + Futures.immediateFuture( + ImmutableMap.of(isolatedStructureGroup.getFile(0), onDeviceUri1))); + + MobileDataDownload mobileDataDownload = + new MobileDataDownloadImpl( + context, + mockEventLogger, + mockMobileDataDownloadManager, + controlExecutor, + ImmutableList.of() /* fileGroupPopulatorList */, + Optional.of(mockTaskScheduler), + fileStorage, + Optional.absent() /* downloadMonitorOptional */, + Optional.of(this.getClass()), // don't need to use the real foreground download service. + flags, + singleFileDownloader, + Optional.absent() /* customFileGroupValidator */, + timeSource); + + ClientFileGroup clientFileGroup = + mobileDataDownload + .getFileGroup( + GetFileGroupRequest.newBuilder() + .setGroupName(FILE_GROUP_NAME_1) + .setVerifyIsolatedStructure(false) + .build()) + .get(); + + assertThat(clientFileGroup.getGroupName()).isEqualTo(FILE_GROUP_NAME_1); + assertThat(clientFileGroup.getFile(0).getFileUri()).isEqualTo(onDeviceUri1.toString()); + + // Verify getting the file uri bypassed the verify check. + verify(mockMobileDataDownloadManager, never()).getDataFileUris(any(), eq(true)); + } + + @Test + public void getFileGroup_whenVerifyIsolatedStructureIsTrue_returnsNullOnInvalidStructure() + throws Exception { + MetadataProto.DataFile isolatedStructureFile = + MetadataProto.DataFile.newBuilder() + .setFileId(FILE_ID_1) + .setChecksumType(MetadataProto.DataFile.ChecksumType.NONE) + .setUrlToDownload(FILE_URL_1) + .setRelativeFilePath("mycustom/file.txt") + .build(); + DataFileGroupInternal isolatedStructureGroup = + DataFileGroupInternal.newBuilder() + .setGroupName(FILE_GROUP_NAME_1) + .setOwnerPackage(context.getPackageName()) + .setPreserveFilenamesAndIsolateFiles(true) + .addFile(isolatedStructureFile) + .build(); + + when(mockMobileDataDownloadManager.getFileGroup(any(GroupKey.class), eq(true))) + .thenReturn(Futures.immediateFuture(isolatedStructureGroup)); + + // Mock that verification failed + when(mockMobileDataDownloadManager.getDataFileUris( + isolatedStructureGroup, /* verifyIsolatedStructure= */ true)) + .thenReturn(Futures.immediateFuture(ImmutableMap.of())); + + MobileDataDownload mobileDataDownload = + new MobileDataDownloadImpl( + context, + mockEventLogger, + mockMobileDataDownloadManager, + controlExecutor, + ImmutableList.of() /* fileGroupPopulatorList */, + Optional.of(mockTaskScheduler), + fileStorage, + Optional.absent() /* downloadMonitorOptional */, + Optional.of(this.getClass()), // don't need to use the real foreground download service. + flags, + singleFileDownloader, + Optional.absent() /* customFileGroupValidator */, + timeSource); + + // Assert that a failure to verify the isolated structure returns a null group + assertThat( + mobileDataDownload + .getFileGroup( + GetFileGroupRequest.newBuilder() + .setGroupName(FILE_GROUP_NAME_1) + .setVerifyIsolatedStructure(true) + .build()) + .get()) + .isNull(); + + // Verify getting the file uri did not bypass the verify check. + verify(mockMobileDataDownloadManager, never()).getDataFileUris(any(), eq(false)); + } + + @Test + public void getFileGroup_fileGroupFound_logsQueryStatsForFileGroup() throws Exception { + DataFileGroupInternal dataFileGroup = + FILE_GROUP_INTERNAL_1.toBuilder().setBuildId(10).setVariantId("test-variant").build(); + when(mockMobileDataDownloadManager.getFileGroup( + any(GroupKey.class), /* downloaded= */ eq(true))) + .thenReturn(Futures.immediateFuture(dataFileGroup)); + when(mockMobileDataDownloadManager.getDataFileUris( + dataFileGroup, /* verifyIsolatedStructure= */ true)) + .thenReturn( + Futures.immediateFuture(ImmutableMap.of(dataFileGroup.getFile(0), onDeviceUri1))); + + MobileDataDownload mobileDataDownload = + new MobileDataDownloadImpl( + context, + fakeEventLogger, + mockMobileDataDownloadManager, + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), + Optional.of(mockTaskScheduler), + fileStorage, + /* downloadMonitorOptional= */ Optional.absent(), + Optional.of(this.getClass()), // don't need to use the real foreground download service. + flags, + singleFileDownloader, + /* customValidatorOptional= */ Optional.absent(), + timeSource); + + ClientFileGroup unused = + mobileDataDownload + .getFileGroup(GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME_1).build()) + .get(); + + List<DataDownloadFileGroupStats> fileGroupDetailsList = + fakeEventLogger.getLoggedMddQueryStats(); + + assertThat(fileGroupDetailsList).hasSize(1); + DataDownloadFileGroupStats fileGroupStats = fileGroupDetailsList.get(0); + assertThat(fileGroupStats.getFileGroupName()).isEqualTo(FILE_GROUP_NAME_1); + assertThat(fileGroupStats.getOwnerPackage()).isEqualTo(context.getPackageName()); + assertThat(fileGroupStats.getFileGroupVersionNumber()).isEqualTo(5); + assertThat(fileGroupStats.getBuildId()).isEqualTo(10); + assertThat(fileGroupStats.getVariantId()).isEqualTo("test-variant"); + assertThat(fileGroupStats.getFileCount()).isEqualTo(1); + } + + @Test + public void getFileGroup_fileGroupFound_doesNotOverLog() throws Exception { + DataFileGroupInternal dataFileGroup = FILE_GROUP_INTERNAL_1; + when(mockMobileDataDownloadManager.getFileGroup( + any(GroupKey.class), /* downloaded= */ eq(true))) + .thenReturn(Futures.immediateFuture(dataFileGroup)); + when(mockMobileDataDownloadManager.getDataFileUris( + dataFileGroup, /* verifyIsolatedStructure= */ true)) + .thenReturn( + Futures.immediateFuture(ImmutableMap.of(dataFileGroup.getFile(0), onDeviceUri1))); + + MobileDataDownload mobileDataDownload = + new MobileDataDownloadImpl( + context, + mockEventLogger, + mockMobileDataDownloadManager, + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), + Optional.of(mockTaskScheduler), + fileStorage, + /* downloadMonitorOptional= */ Optional.absent(), + Optional.of(this.getClass()), // don't need to use the real foreground download service. + flags, + singleFileDownloader, + /* customValidatorOptional= */ Optional.absent(), + timeSource); + + ClientFileGroup unused = + mobileDataDownload + .getFileGroup(GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME_1).build()) + .get(); + + verify(mockEventLogger).logMddQueryStats(any()); + verify(mockEventLogger).logMddLibApiResultLog(any()); + verifyNoMoreInteractions(mockEventLogger); + } + + @Test + public void getFileGroup_fileGroupNotFound_doesNotOverLog() throws Exception { + when(mockMobileDataDownloadManager.getFileGroup( + any(GroupKey.class), /* downloaded= */ eq(true))) + .thenReturn(Futures.immediateFuture(null)); + + MobileDataDownload mobileDataDownload = + new MobileDataDownloadImpl( + context, + mockEventLogger, + mockMobileDataDownloadManager, + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), + Optional.of(mockTaskScheduler), + fileStorage, + /* downloadMonitorOptional= */ Optional.absent(), + Optional.of(this.getClass()), // don't need to use the real foreground download service. + flags, + singleFileDownloader, + /* customValidatorOptional= */ Optional.absent(), + timeSource); + + ClientFileGroup unused = + mobileDataDownload + .getFileGroup(GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME_1).build()) + .get(); + + verify(mockEventLogger).logMddLibApiResultLog(any()); + verifyNoMoreInteractions(mockEventLogger); + } + + @Test + public void getFileGroup_throwsException_doesNotOverLog() throws Exception { + when(mockMobileDataDownloadManager.getFileGroup(any(GroupKey.class), eq(true))) + .thenReturn(Futures.immediateFailedFuture(new Exception())); + + MobileDataDownload mobileDataDownload = + new MobileDataDownloadImpl( + context, + mockEventLogger, + mockMobileDataDownloadManager, + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), + Optional.of(mockTaskScheduler), + fileStorage, + /* downloadMonitorOptional= */ Optional.absent(), + Optional.of(this.getClass()), // don't need to use the real foreground download service. + flags, + singleFileDownloader, + /* customValidatorOptional= */ Optional.absent(), + timeSource); + + assertThrows( + ExecutionException.class, + () -> + mobileDataDownload + .getFileGroup( + GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME_1).build()) + .get()); + + verify(mockEventLogger).logMddLibApiResultLog(any()); + verifyNoMoreInteractions(mockEventLogger); + } + + /** + * Helper function to test that expected errors are being logged. + * + * <p>causeThrowable is used to check for cause only if expectedThrowable is instance of + * ExecutionException. + */ + private <T extends Throwable> void getFileGroupErrorLoggingTestHelper( + ListenableFuture<?> getFileGroupResultFuture, + Class<T> expectedThrowable, + Class<?> causeThrowable, + int code) + throws Exception { + long latencyNs = 1000; + when(mockMobileDataDownloadManager.getFileGroup( + any(GroupKey.class), /* downloaded= */ eq(true))) + .thenAnswer( + invocation -> { + // Advancing time source to test latency. + timeSource.advance(latencyNs, TimeUnit.NANOSECONDS); + return getFileGroupResultFuture; + }); + + MobileDataDownload mobileDataDownload = + new MobileDataDownloadImpl( + context, + fakeEventLogger, + mockMobileDataDownloadManager, + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), + Optional.of(mockTaskScheduler), + fileStorage, + /* downloadMonitorOptional= */ Optional.absent(), + Optional.of(this.getClass()), // don't need to use the real foreground download service. + flags, + singleFileDownloader, + /* customValidatorOptional= */ Optional.absent(), + timeSource); + + Throwable thrown = + assertThrows( + expectedThrowable, + () -> + mobileDataDownload + .getFileGroup( + GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME_1).build()) + .get()); + + if (thrown instanceof ExecutionException) { + assertThat(thrown).hasCauseThat().isInstanceOf(causeThrowable); + } + } + + @Test + public void getFileGroup_throwsCancelledException_logsCancelled() throws Exception { + getFileGroupErrorLoggingTestHelper( + Futures.immediateCancelledFuture(), CancellationException.class, null, 0); + } + + @Test + public void getFileGroup_throwsUnknownException_logsFailureWithoutCause() throws Exception { + getFileGroupErrorLoggingTestHelper( + Futures.immediateFailedFuture(new Exception()), + ExecutionException.class, + Exception.class, + 0); + } + + @Test + public void getFileGroup_throwsInterruptedException_logsInterrupted() throws Exception { + getFileGroupErrorLoggingTestHelper( + Futures.immediateFailedFuture(new InterruptedException()), + ExecutionException.class, + InterruptedException.class, + 0); + } + + @Test + public void getFileGroup_throwsIOException_logsIOError() throws Exception { + getFileGroupErrorLoggingTestHelper( + Futures.immediateFailedFuture(new IOException()), + ExecutionException.class, + IOException.class, + 0); + } + + @Test + public void getFileGroup_throwsIllegalStateException_logsIllegalState() throws Exception { + getFileGroupErrorLoggingTestHelper( + Futures.immediateFailedFuture(new IllegalStateException()), + ExecutionException.class, + IllegalStateException.class, + 0); + } + + @Test + public void getFileGroup_throwsIllegalArgumentException_logsIllegalArgument() throws Exception { + getFileGroupErrorLoggingTestHelper( + Futures.immediateFailedFuture(new IllegalArgumentException()), + ExecutionException.class, + IllegalArgumentException.class, + 0); + } + + @Test + public void getFileGroup_throwsUnsupportedOperationException_logsUnsupportedOperation() + throws Exception { + getFileGroupErrorLoggingTestHelper( + Futures.immediateFailedFuture(new UnsupportedOperationException()), + ExecutionException.class, + UnsupportedOperationException.class, + 0); + } + + @Test + public void getFileGroup_throwsDownloadException_logsDownloadError() throws Exception { + getFileGroupErrorLoggingTestHelper( + Futures.immediateFailedFuture( + DownloadException.builder() + .setDownloadResultCode(DownloadResultCode.UNSPECIFIED) + .build()), + ExecutionException.class, + DownloadException.class, + 0); + } + + @Test + public void readDataFileGroup_returnsFileGroup() throws Exception { + DataFileGroupInternal dataFileGroupInternal = + DataFileGroupInternal.newBuilder() + .setGroupName(FILE_GROUP_NAME_1) + .setOwnerPackage(context.getPackageName()) + .addFile( + MetadataProto.DataFile.newBuilder() + .setFileId(FILE_ID_1) + .setUrlToDownload(FILE_URL_1) + .build()) + .addFile( + MetadataProto.DataFile.newBuilder() + .setFileId(FILE_ID_2) + .setUrlToDownload(FILE_URL_2) + .build()) + .build(); + + when(mockMobileDataDownloadManager.getFileGroup(any(GroupKey.class), eq(true))) + .thenReturn(Futures.immediateFuture(dataFileGroupInternal)); + when(mockMobileDataDownloadManager.getDataFileUris( + dataFileGroupInternal, /* verifyIsolatedStructure= */ true)) + .thenReturn( + Futures.immediateFuture( + ImmutableMap.of( + dataFileGroupInternal.getFile(0), + onDeviceUri1, + dataFileGroupInternal.getFile(1), + onDeviceUri1))); + + MobileDataDownload mobileDataDownload = + new MobileDataDownloadImpl( + context, + mockEventLogger, + mockMobileDataDownloadManager, + controlExecutor, + ImmutableList.of() /* fileGroupPopulatorList */, + Optional.of(mockTaskScheduler), + fileStorage, + Optional.absent() /* downloadMonitorOptional */, + Optional.of(this.getClass()), // don't need to use the real foreground download service. + flags, + singleFileDownloader, + Optional.absent() /* customFileGroupValidator */, + timeSource); + + DataFileGroup dataFileGroup = + mobileDataDownload + .readDataFileGroup( + ReadDataFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME_1).build()) + .get(); + + assertThat(dataFileGroup.getGroupName()).isEqualTo(FILE_GROUP_NAME_1); + assertThat(dataFileGroup.getFileList()) + .containsExactly( + DataFile.newBuilder().setFileId(FILE_ID_1).setUrlToDownload(FILE_URL_1).build(), + DataFile.newBuilder().setFileId(FILE_ID_2).setUrlToDownload(FILE_URL_2).build()); + } + + @Test public void getFileGroupsByFilter_singleGroup() throws Exception { - List<Pair<GroupKey, DataFileGroupInternal>> keyDataFileGroupList = new ArrayList<>(); + List<GroupKeyAndGroup> keyDataFileGroupList = new ArrayList<>(); - DataFileGroupInternal downloadedFileGroup = - createDataFileGroupInternal( - FILE_GROUP_NAME_1, - context.getPackageName(), - 5 /* versionNumber */, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI); + DataFileGroupInternal downloadedFileGroup = FILE_GROUP_INTERNAL_1; GroupKey groupKey = GroupKey.newBuilder() @@ -1397,11 +1720,13 @@ public class MobileDataDownloadTest { .build(); when(mockMobileDataDownloadManager.getFileGroup(eq(groupKey), eq(true))) .thenReturn(Futures.immediateFuture(downloadedFileGroup)); - when(mockMobileDataDownloadManager.getDataFileUri( - downloadedFileGroup.getFile(0), downloadedFileGroup)) - .thenReturn(Futures.immediateFuture(onDeviceUri1)); + when(mockMobileDataDownloadManager.getDataFileUris( + downloadedFileGroup, /* verifyIsolatedStructure= */ true)) + .thenReturn( + Futures.immediateFuture(ImmutableMap.of(downloadedFileGroup.getFile(0), onDeviceUri1))); keyDataFileGroupList.add( - Pair.create(groupKey.toBuilder().setDownloaded(true).build(), downloadedFileGroup)); + GroupKeyAndGroup.create( + groupKey.toBuilder().setDownloaded(true).build(), downloadedFileGroup)); DataFileGroupInternal pendingFileGroup = createDataFileGroupInternal( @@ -1419,8 +1744,12 @@ public class MobileDataDownloadTest { .build(); when(mockMobileDataDownloadManager.getFileGroup(groupKey, false)) .thenReturn(Futures.immediateFuture(pendingFileGroup)); + when(mockMobileDataDownloadManager.getDataFileUris( + pendingFileGroup, /* verifyIsolatedStructure= */ true)) + .thenReturn(Futures.immediateFuture(ImmutableMap.of())); keyDataFileGroupList.add( - Pair.create(groupKey.toBuilder().setDownloaded(false).build(), pendingFileGroup)); + GroupKeyAndGroup.create( + groupKey.toBuilder().setDownloaded(false).build(), pendingFileGroup)); DataFileGroupInternal pendingFileGroup2 = createDataFileGroupInternal( @@ -1439,8 +1768,12 @@ public class MobileDataDownloadTest { .build(); when(mockMobileDataDownloadManager.getFileGroup(groupKey2, false)) .thenReturn(Futures.immediateFuture(pendingFileGroup2)); + when(mockMobileDataDownloadManager.getDataFileUris( + pendingFileGroup2, /* verifyIsolatedStructure= */ true)) + .thenReturn(Futures.immediateFuture(ImmutableMap.of())); keyDataFileGroupList.add( - Pair.create(groupKey2.toBuilder().setDownloaded(false).build(), pendingFileGroup2)); + GroupKeyAndGroup.create( + groupKey2.toBuilder().setDownloaded(false).build(), pendingFileGroup2)); when(mockMobileDataDownloadManager.getAllFreshGroups()) .thenReturn(Futures.immediateFuture(keyDataFileGroupList)); @@ -1450,7 +1783,7 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -1458,7 +1791,8 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); // We should get back 2 groups for FILE_GROUP_NAME_1. GetFileGroupsByFilterRequest getFileGroupsByFilterRequest = @@ -1508,35 +1842,49 @@ public class MobileDataDownloadTest { assertThat(pendingClientFileGroup2.getStatus()).isEqualTo(Status.PENDING); assertThat(pendingClientFileGroup2.getFileCount()).isEqualTo(2); assertThat(pendingClientFileGroup2.hasAccount()).isFalse(); + + ArgumentCaptor<DataDownloadFileGroupStats> fileGroupDetailsCaptor = + ArgumentCaptor.forClass(DataDownloadFileGroupStats.class); + verify(mockEventLogger, times(3)).logMddQueryStats(fileGroupDetailsCaptor.capture()); + + List<DataDownloadFileGroupStats> fileGroupStats = fileGroupDetailsCaptor.getAllValues(); + assertThat(fileGroupStats).hasSize(3); + assertThat(fileGroupStats.get(0).getFileGroupName()).isEqualTo(FILE_GROUP_NAME_1); + assertThat(fileGroupStats.get(0).getOwnerPackage()).isEqualTo(context.getPackageName()); + assertThat(fileGroupStats.get(0).getFileGroupVersionNumber()).isEqualTo(5); + assertThat(fileGroupStats.get(0).getFileCount()).isEqualTo(1); + assertThat(fileGroupStats.get(1).getFileGroupName()).isEqualTo(FILE_GROUP_NAME_1); + assertThat(fileGroupStats.get(1).getOwnerPackage()).isEqualTo(context.getPackageName()); + assertThat(fileGroupStats.get(1).getFileGroupVersionNumber()).isEqualTo(7); + assertThat(fileGroupStats.get(1).getFileCount()).isEqualTo(1); + assertThat(fileGroupStats.get(2).getFileGroupName()).isEqualTo(FILE_GROUP_NAME_2); + assertThat(fileGroupStats.get(2).getOwnerPackage()).isEqualTo(context.getPackageName()); + assertThat(fileGroupStats.get(2).getFileGroupVersionNumber()).isEqualTo(4); + assertThat(fileGroupStats.get(2).getFileCount()).isEqualTo(2); + + verifyNoMoreInteractions(mockEventLogger); } @Test public void getFileGroupsByFilter_includeAllGroups() throws Exception { - List<Pair<GroupKey, DataFileGroupInternal>> keyDataFileGroupList = new ArrayList<>(); + List<GroupKeyAndGroup> keyDataFileGroupList = new ArrayList<>(); Account account = AccountUtil.create("account-name", "account-type"); - DataFileGroupInternal downloadedFileGroup = - createDataFileGroupInternal( - FILE_GROUP_NAME_1, - context.getPackageName(), - 5 /* versionNumber */, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI); + DataFileGroupInternal downloadedFileGroup = FILE_GROUP_INTERNAL_1; GroupKey groupKey = GroupKey.newBuilder() .setGroupName(FILE_GROUP_NAME_1) .setOwnerPackage(context.getPackageName()) .setAccount(AccountUtil.serialize(account)) .build(); - when(mockMobileDataDownloadManager.getDataFileUri( - downloadedFileGroup.getFile(0), downloadedFileGroup)) - .thenReturn(Futures.immediateFuture(onDeviceUri1)); + when(mockMobileDataDownloadManager.getDataFileUris( + downloadedFileGroup, /* verifyIsolatedStructure= */ true)) + .thenReturn( + Futures.immediateFuture(ImmutableMap.of(downloadedFileGroup.getFile(0), onDeviceUri1))); keyDataFileGroupList.add( - Pair.create(groupKey.toBuilder().setDownloaded(true).build(), downloadedFileGroup)); + GroupKeyAndGroup.create( + groupKey.toBuilder().setDownloaded(true).build(), downloadedFileGroup)); DataFileGroupInternal pendingFileGroup = createDataFileGroupInternal( @@ -1548,8 +1896,12 @@ public class MobileDataDownloadTest { new String[] {FILE_CHECKSUM_2}, new String[] {FILE_URL_2}, DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI); + when(mockMobileDataDownloadManager.getDataFileUris( + pendingFileGroup, /* verifyIsolatedStructure= */ true)) + .thenReturn(Futures.immediateFuture(ImmutableMap.of())); keyDataFileGroupList.add( - Pair.create(groupKey.toBuilder().setDownloaded(false).build(), pendingFileGroup)); + GroupKeyAndGroup.create( + groupKey.toBuilder().setDownloaded(false).build(), pendingFileGroup)); DataFileGroupInternal pendingFileGroup2 = createDataFileGroupInternal( @@ -1568,8 +1920,12 @@ public class MobileDataDownloadTest { .build(); when(mockMobileDataDownloadManager.getFileGroup(groupKey2, false)) .thenReturn(Futures.immediateFuture(pendingFileGroup2)); + when(mockMobileDataDownloadManager.getDataFileUris( + pendingFileGroup2, /* verifyIsolatedStructure= */ true)) + .thenReturn(Futures.immediateFuture(ImmutableMap.of())); keyDataFileGroupList.add( - Pair.create(groupKey2.toBuilder().setDownloaded(false).build(), pendingFileGroup2)); + GroupKeyAndGroup.create( + groupKey2.toBuilder().setDownloaded(false).build(), pendingFileGroup2)); when(mockMobileDataDownloadManager.getAllFreshGroups()) .thenReturn(Futures.immediateFuture(keyDataFileGroupList)); @@ -1579,7 +1935,7 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -1587,7 +1943,8 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); // We should get back all 3 groups for this key. GetFileGroupsByFilterRequest getFileGroupsByFilterRequest = @@ -1628,6 +1985,27 @@ public class MobileDataDownloadTest { assertThat(pendingClientFileGroup2.getStatus()).isEqualTo(Status.PENDING); assertThat(pendingClientFileGroup2.getFileCount()).isEqualTo(2); assertThat(pendingClientFileGroup2.hasAccount()).isFalse(); + + ArgumentCaptor<DataDownloadFileGroupStats> fileGroupDetailsCaptor = + ArgumentCaptor.forClass(DataDownloadFileGroupStats.class); + verify(mockEventLogger, times(3)).logMddQueryStats(fileGroupDetailsCaptor.capture()); + + List<DataDownloadFileGroupStats> fileGroupStats = fileGroupDetailsCaptor.getAllValues(); + assertThat(fileGroupStats).hasSize(3); + assertThat(fileGroupStats.get(0).getFileGroupName()).isEqualTo(FILE_GROUP_NAME_1); + assertThat(fileGroupStats.get(0).getOwnerPackage()).isEqualTo(context.getPackageName()); + assertThat(fileGroupStats.get(0).getFileGroupVersionNumber()).isEqualTo(5); + assertThat(fileGroupStats.get(0).getFileCount()).isEqualTo(1); + assertThat(fileGroupStats.get(1).getFileGroupName()).isEqualTo(FILE_GROUP_NAME_1); + assertThat(fileGroupStats.get(1).getOwnerPackage()).isEqualTo(context.getPackageName()); + assertThat(fileGroupStats.get(1).getFileGroupVersionNumber()).isEqualTo(7); + assertThat(fileGroupStats.get(1).getFileCount()).isEqualTo(1); + assertThat(fileGroupStats.get(2).getFileGroupName()).isEqualTo(FILE_GROUP_NAME_2); + assertThat(fileGroupStats.get(2).getOwnerPackage()).isEqualTo(context.getPackageName()); + assertThat(fileGroupStats.get(2).getFileGroupVersionNumber()).isEqualTo(4); + assertThat(fileGroupStats.get(2).getFileCount()).isEqualTo(2); + + verifyNoMoreInteractions(mockEventLogger); } @Test @@ -1637,7 +2015,7 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -1645,7 +2023,8 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); when(mockMobileDataDownloadManager.getAllFreshGroups()) .thenReturn(Futures.immediateFuture(ImmutableList.of())); @@ -1667,21 +2046,12 @@ public class MobileDataDownloadTest { @Test public void getFileGroupsByFilter_withAccount() throws Exception { - List<Pair<GroupKey, DataFileGroupInternal>> keyDataFileGroupList = new ArrayList<>(); + List<GroupKeyAndGroup> keyDataFileGroupList = new ArrayList<>(); Account account1 = AccountUtil.create("account-name-1", "account-type"); Account account2 = AccountUtil.create("account-name-2", "account-type"); - DataFileGroupInternal downloadedFileGroup = - createDataFileGroupInternal( - FILE_GROUP_NAME_1, - context.getPackageName(), - 5 /* versionNumber */, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI); + DataFileGroupInternal downloadedFileGroup = FILE_GROUP_INTERNAL_1; GroupKey groupKey = GroupKey.newBuilder() .setGroupName(FILE_GROUP_NAME_1) @@ -1690,11 +2060,13 @@ public class MobileDataDownloadTest { .build(); when(mockMobileDataDownloadManager.getFileGroup(groupKey, true)) .thenReturn(Futures.immediateFuture(downloadedFileGroup)); - when(mockMobileDataDownloadManager.getDataFileUri( - downloadedFileGroup.getFile(0), downloadedFileGroup)) - .thenReturn(Futures.immediateFuture(onDeviceUri1)); + when(mockMobileDataDownloadManager.getDataFileUris( + downloadedFileGroup, /* verifyIsolatedStructure= */ true)) + .thenReturn( + Futures.immediateFuture(ImmutableMap.of(downloadedFileGroup.getFile(0), onDeviceUri1))); keyDataFileGroupList.add( - Pair.create(groupKey.toBuilder().setDownloaded(true).build(), downloadedFileGroup)); + GroupKeyAndGroup.create( + groupKey.toBuilder().setDownloaded(true).build(), downloadedFileGroup)); DataFileGroupInternal pendingFileGroup = createDataFileGroupInternal( @@ -1708,8 +2080,12 @@ public class MobileDataDownloadTest { DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI); when(mockMobileDataDownloadManager.getFileGroup(groupKey, false)) .thenReturn(Futures.immediateFuture(pendingFileGroup)); + when(mockMobileDataDownloadManager.getDataFileUris( + pendingFileGroup, /* verifyIsolatedStructure= */ true)) + .thenReturn(Futures.immediateFuture(ImmutableMap.of())); keyDataFileGroupList.add( - Pair.create(groupKey.toBuilder().setDownloaded(false).build(), pendingFileGroup)); + GroupKeyAndGroup.create( + groupKey.toBuilder().setDownloaded(false).build(), pendingFileGroup)); DataFileGroupInternal pendingFileGroup2 = createDataFileGroupInternal( @@ -1729,8 +2105,12 @@ public class MobileDataDownloadTest { .build(); when(mockMobileDataDownloadManager.getFileGroup(groupKey2, false)) .thenReturn(Futures.immediateFuture(pendingFileGroup2)); + when(mockMobileDataDownloadManager.getDataFileUris( + pendingFileGroup2, /* verifyIsolatedStructure= */ true)) + .thenReturn(Futures.immediateFuture(ImmutableMap.of())); keyDataFileGroupList.add( - Pair.create(groupKey2.toBuilder().setDownloaded(false).build(), pendingFileGroup2)); + GroupKeyAndGroup.create( + groupKey2.toBuilder().setDownloaded(false).build(), pendingFileGroup2)); when(mockMobileDataDownloadManager.getAllFreshGroups()) .thenReturn(Futures.immediateFuture(keyDataFileGroupList)); @@ -1740,7 +2120,7 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -1748,7 +2128,8 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); // We should get back 2 groups for FILE_GROUP_NAME_1 with account1. GetFileGroupsByFilterRequest getFileGroupsByFilterRequest = @@ -1799,26 +2180,38 @@ public class MobileDataDownloadTest { assertThat(pendingClientFileGroup2.getAccount()).isEqualTo(AccountUtil.serialize(account2)); assertThat(pendingClientFileGroup2.getStatus()).isEqualTo(Status.PENDING); assertThat(pendingClientFileGroup2.getFileCount()).isEqualTo(2); + + ArgumentCaptor<DataDownloadFileGroupStats> fileGroupDetailsCaptor = + ArgumentCaptor.forClass(DataDownloadFileGroupStats.class); + verify(mockEventLogger, times(3)).logMddQueryStats(fileGroupDetailsCaptor.capture()); + + List<DataDownloadFileGroupStats> fileGroupStats = fileGroupDetailsCaptor.getAllValues(); + assertThat(fileGroupStats).hasSize(3); + assertThat(fileGroupStats.get(0).getFileGroupName()).isEqualTo(FILE_GROUP_NAME_1); + assertThat(fileGroupStats.get(0).getOwnerPackage()).isEqualTo(context.getPackageName()); + assertThat(fileGroupStats.get(0).getFileGroupVersionNumber()).isEqualTo(5); + assertThat(fileGroupStats.get(0).getFileCount()).isEqualTo(1); + assertThat(fileGroupStats.get(1).getFileGroupName()).isEqualTo(FILE_GROUP_NAME_1); + assertThat(fileGroupStats.get(1).getOwnerPackage()).isEqualTo(context.getPackageName()); + assertThat(fileGroupStats.get(1).getFileGroupVersionNumber()).isEqualTo(7); + assertThat(fileGroupStats.get(1).getFileCount()).isEqualTo(1); + assertThat(fileGroupStats.get(2).getFileGroupName()).isEqualTo(FILE_GROUP_NAME_1); + assertThat(fileGroupStats.get(2).getOwnerPackage()).isEqualTo(context.getPackageName()); + assertThat(fileGroupStats.get(2).getFileGroupVersionNumber()).isEqualTo(4); + assertThat(fileGroupStats.get(2).getFileCount()).isEqualTo(2); + + verifyNoMoreInteractions(mockEventLogger); } @Test public void getFileGroupsByFilter_groupWithNoAccountOnly() throws Exception { - List<Pair<GroupKey, DataFileGroupInternal>> keyDataFileGroupList = new ArrayList<>(); + List<GroupKeyAndGroup> keyDataFileGroupList = new ArrayList<>(); Account account1 = AccountUtil.create("account-name-1", "account-type"); Account account2 = AccountUtil.create("account-name-2", "account-type"); // downloadedFileGroup is associated with account1. - DataFileGroupInternal downloadedFileGroup = - createDataFileGroupInternal( - FILE_GROUP_NAME_1, - context.getPackageName(), - /*versionNumber=*/ 5, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI); + DataFileGroupInternal downloadedFileGroup = FILE_GROUP_INTERNAL_1; GroupKey groupKey = GroupKey.newBuilder() .setGroupName(FILE_GROUP_NAME_1) @@ -1827,18 +2220,20 @@ public class MobileDataDownloadTest { .build(); when(mockMobileDataDownloadManager.getFileGroup(groupKey, true)) .thenReturn(Futures.immediateFuture(downloadedFileGroup)); - when(mockMobileDataDownloadManager.getDataFileUri( - downloadedFileGroup.getFile(0), downloadedFileGroup)) - .thenReturn(Futures.immediateFuture(onDeviceUri1)); + when(mockMobileDataDownloadManager.getDataFileUris( + downloadedFileGroup, /* verifyIsolatedStructure= */ true)) + .thenReturn( + Futures.immediateFuture(ImmutableMap.of(downloadedFileGroup.getFile(0), onDeviceUri1))); keyDataFileGroupList.add( - Pair.create(groupKey.toBuilder().setDownloaded(true).build(), downloadedFileGroup)); + GroupKeyAndGroup.create( + groupKey.toBuilder().setDownloaded(true).build(), downloadedFileGroup)); // pendingFileGroup is associated with account2. DataFileGroupInternal pendingFileGroup = createDataFileGroupInternal( FILE_GROUP_NAME_1, context.getPackageName(), - /*versionNumber=*/ 7, + /* versionNumber= */ 7, new String[] {FILE_ID_1}, new int[] {FILE_SIZE_2}, new String[] {FILE_CHECKSUM_2}, @@ -1852,15 +2247,19 @@ public class MobileDataDownloadTest { .build(); when(mockMobileDataDownloadManager.getFileGroup(groupKey2, false)) .thenReturn(Futures.immediateFuture(pendingFileGroup)); + when(mockMobileDataDownloadManager.getDataFileUris( + pendingFileGroup, /* verifyIsolatedStructure= */ true)) + .thenReturn(Futures.immediateFuture(ImmutableMap.of())); keyDataFileGroupList.add( - Pair.create(groupKey2.toBuilder().setDownloaded(false).build(), pendingFileGroup)); + GroupKeyAndGroup.create( + groupKey2.toBuilder().setDownloaded(false).build(), pendingFileGroup)); // pendingFileGroup2 is an account independent group. DataFileGroupInternal pendingFileGroup2 = createDataFileGroupInternal( FILE_GROUP_NAME_1, context.getPackageName(), - /*versionNumber=*/ 4, + /* versionNumber= */ 4, new String[] {FILE_ID_1, FILE_ID_2}, new int[] {FILE_SIZE_1, FILE_SIZE_2}, new String[] {FILE_CHECKSUM_1, FILE_CHECKSUM_2}, @@ -1873,8 +2272,12 @@ public class MobileDataDownloadTest { .build(); when(mockMobileDataDownloadManager.getFileGroup(groupKey3, false)) .thenReturn(Futures.immediateFuture(pendingFileGroup2)); + when(mockMobileDataDownloadManager.getDataFileUris( + pendingFileGroup2, /* verifyIsolatedStructure= */ true)) + .thenReturn(Futures.immediateFuture(ImmutableMap.of())); keyDataFileGroupList.add( - Pair.create(groupKey3.toBuilder().setDownloaded(false).build(), pendingFileGroup2)); + GroupKeyAndGroup.create( + groupKey3.toBuilder().setDownloaded(false).build(), pendingFileGroup2)); when(mockMobileDataDownloadManager.getAllFreshGroups()) .thenReturn(Futures.immediateFuture(keyDataFileGroupList)); @@ -1884,15 +2287,16 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, - /*fileGroupPopulatorList=*/ ImmutableList.of(), + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), Optional.of(mockTaskScheduler), fileStorage, - /*downloadMonitorOptional=*/ Optional.absent(), - /* foregroundDownloadServiceClassOptional = */ Optional.absent(), + /* downloadMonitorOptional= */ Optional.absent(), + /* foregroundDownloadServiceClassOptional= */ Optional.absent(), flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); // We should get back only 1 group for FILE_GROUP_NAME_1 with groupWithNoAccountOnly being set // to true. @@ -1912,6 +2316,19 @@ public class MobileDataDownloadTest { assertThat(pendingClientFileGroup.getStatus()).isEqualTo(Status.PENDING); assertThat(pendingClientFileGroup.getFileCount()).isEqualTo(2); assertThat(pendingClientFileGroup.hasAccount()).isFalse(); + + ArgumentCaptor<DataDownloadFileGroupStats> fileGroupDetailsCaptor = + ArgumentCaptor.forClass(DataDownloadFileGroupStats.class); + verify(mockEventLogger, times(1)).logMddQueryStats(fileGroupDetailsCaptor.capture()); + + List<DataDownloadFileGroupStats> fileGroupStats = fileGroupDetailsCaptor.getAllValues(); + assertThat(fileGroupStats).hasSize(1); + assertThat(fileGroupStats.get(0).getFileGroupName()).isEqualTo(FILE_GROUP_NAME_1); + assertThat(fileGroupStats.get(0).getOwnerPackage()).isEqualTo(context.getPackageName()); + assertThat(fileGroupStats.get(0).getFileGroupVersionNumber()).isEqualTo(4); + assertThat(fileGroupStats.get(0).getFileCount()).isEqualTo(2); + + verifyNoMoreInteractions(mockEventLogger); } @Test @@ -1939,15 +2356,16 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, - /* fileGroupPopulatorList = */ ImmutableList.of(), + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), Optional.of(mockTaskScheduler), fileStorage, Optional.of(mockDownloadMonitor), Optional.of(this.getClass()), flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); // Since we use mocks, just call the method directly, no need to call addFileGroup first mobileDataDownload @@ -2006,15 +2424,16 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, - /* fileGroupPopulatorList = */ ImmutableList.of(), + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), Optional.of(mockTaskScheduler), fileStorage, Optional.of(mockDownloadMonitor), Optional.of(this.getClass()), flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); // Since we use mocks, just call the method directly, no need to call addFileGroup first mobileDataDownload @@ -2074,15 +2493,16 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, - /* fileGroupPopulatorList = */ ImmutableList.of(), + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), Optional.of(mockTaskScheduler), fileStorage, Optional.of(mockDownloadMonitor), Optional.of(this.getClass()), flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); // Since we use mocks, just call the method directly, no need to call addFileGroup first ExecutionException ex = @@ -2122,28 +2542,25 @@ public class MobileDataDownloadTest { @Test public void downloadFileGroup() throws Exception { - DataFileGroupInternal dataFileGroup = - createDataFileGroupInternal( - FILE_GROUP_NAME_1, - context.getPackageName(), - 5 /* versionNumber */, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI); ArgumentCaptor<GroupKey> groupKeyCaptor = ArgumentCaptor.forClass(GroupKey.class); when(mockMobileDataDownloadManager.downloadFileGroup(groupKeyCaptor.capture(), any(), any())) - .thenReturn(Futures.immediateFuture(dataFileGroup)); - when(mockMobileDataDownloadManager.getDataFileUri(dataFileGroup.getFile(0), dataFileGroup)) - .thenReturn(Futures.immediateFuture(onDeviceUri1)); + .thenReturn(Futures.immediateFuture(FILE_GROUP_INTERNAL_1)); + when(mockMobileDataDownloadManager.getFileGroup(any(), eq(false))) + .thenReturn(Futures.immediateFuture(FILE_GROUP_INTERNAL_1)); + when(mockMobileDataDownloadManager.getFileGroup(any(), eq(true))) + .thenReturn(Futures.immediateFuture(null)); + when(mockMobileDataDownloadManager.getDataFileUris( + FILE_GROUP_INTERNAL_1, /* verifyIsolatedStructure= */ true)) + .thenReturn( + Futures.immediateFuture( + ImmutableMap.of(FILE_GROUP_INTERNAL_1.getFile(0), onDeviceUri1))); MobileDataDownload mobileDataDownload = new MobileDataDownloadImpl( context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -2151,9 +2568,10 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); - CountDownLatch onCompleteLatch = new CountDownLatch(1); + AtomicBoolean onCompleteInvoked = new AtomicBoolean(); ClientFileGroup clientFileGroup = mobileDataDownload @@ -2168,25 +2586,19 @@ public class MobileDataDownloadTest { @Override public void onComplete(ClientFileGroup clientFileGroup) { + onCompleteInvoked.set(true); assertThat(clientFileGroup.getGroupName()) .isEqualTo(FILE_GROUP_NAME_1); assertThat(clientFileGroup.getOwnerPackage()) .isEqualTo(context.getPackageName()); assertThat(clientFileGroup.getVersionNumber()).isEqualTo(5); assertThat(clientFileGroup.getFileCount()).isEqualTo(1); - - // This is to verify that onComplete is called. - onCompleteLatch.countDown(); } })) .build()) .get(); - // Verify that onComplete is called. - if (!onCompleteLatch.await(LATCH_WAIT_TIME_MS, TimeUnit.MILLISECONDS)) { - throw new RuntimeException("onComplete is not called"); - } - + assertThat(onCompleteInvoked.get()).isTrue(); assertThat(clientFileGroup.getGroupName()).isEqualTo(FILE_GROUP_NAME_1); assertThat(clientFileGroup.getOwnerPackage()).isEqualTo(context.getPackageName()); assertThat(clientFileGroup.getVersionNumber()).isEqualTo(5); @@ -2209,6 +2621,10 @@ public class MobileDataDownloadTest { @Test public void downloadFileGroup_failed() throws Exception { ArgumentCaptor<GroupKey> groupKeyCaptor = ArgumentCaptor.forClass(GroupKey.class); + when(mockMobileDataDownloadManager.getFileGroup(any(), eq(false))) + .thenReturn(Futures.immediateFuture(FILE_GROUP_INTERNAL_1)); + when(mockMobileDataDownloadManager.getFileGroup(any(), eq(true))) + .thenReturn(Futures.immediateFuture(null)); when(mockMobileDataDownloadManager.downloadFileGroup(groupKeyCaptor.capture(), any(), any())) .thenReturn( Futures.immediateFailedFuture( @@ -2222,7 +2638,7 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -2230,7 +2646,11 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); + + AtomicBoolean listenerOnFailureInvoked = new AtomicBoolean(); + AtomicBoolean callbackOnFailureInvoked = new AtomicBoolean(); ListenableFuture<ClientFileGroup> downloadFuture = mobileDataDownload.downloadFileGroup( @@ -2244,11 +2664,14 @@ public class MobileDataDownloadTest { @Override public void onComplete(ClientFileGroup clientFileGroup) {} + + @Override + public void onFailure(Throwable t) { + listenerOnFailureInvoked.set(true); + } })) .build()); - CountDownLatch onFailureLatch = new CountDownLatch(1); - Futures.addCallback( downloadFuture, new FutureCallback<ClientFileGroup>() { @@ -2257,8 +2680,7 @@ public class MobileDataDownloadTest { @Override public void onFailure(Throwable t) { - // This is to ensure that onFailure is called. - onFailureLatch.countDown(); + callbackOnFailureInvoked.set(true); } }, MoreExecutors.directExecutor()); @@ -2267,10 +2689,8 @@ public class MobileDataDownloadTest { DownloadException e = LabsFutures.getFailureCauseAs(downloadFuture, DownloadException.class); assertThat(e).hasMessageThat().contains("Fail"); - if (!onFailureLatch.await(LATCH_WAIT_TIME_MS, TimeUnit.MILLISECONDS)) { - throw new RuntimeException("latch timeout: onFailure is not called"); - } - + assertThat(listenerOnFailureInvoked.get()).isTrue(); + assertThat(callbackOnFailureInvoked.get()).isTrue(); verify(mockMobileDataDownloadManager).downloadFileGroup(any(GroupKey.class), any(), any()); verify(mockDownloadMonitor) .addDownloadListener(eq(FILE_GROUP_NAME_1), any(DownloadListener.class)); @@ -2282,28 +2702,25 @@ public class MobileDataDownloadTest { @Test public void downloadFileGroup_withAccount() throws Exception { - DataFileGroupInternal dataFileGroup = - createDataFileGroupInternal( - FILE_GROUP_NAME_1, - context.getPackageName(), - 5 /* versionNumber */, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI); ArgumentCaptor<GroupKey> groupKeyCaptor = ArgumentCaptor.forClass(GroupKey.class); when(mockMobileDataDownloadManager.downloadFileGroup(groupKeyCaptor.capture(), any(), any())) - .thenReturn(Futures.immediateFuture(dataFileGroup)); - when(mockMobileDataDownloadManager.getDataFileUri(dataFileGroup.getFile(0), dataFileGroup)) - .thenReturn(Futures.immediateFuture(onDeviceUri1)); + .thenReturn(Futures.immediateFuture(FILE_GROUP_INTERNAL_1)); + when(mockMobileDataDownloadManager.getFileGroup(any(), eq(false))) + .thenReturn(Futures.immediateFuture(FILE_GROUP_INTERNAL_1)); + when(mockMobileDataDownloadManager.getFileGroup(any(), eq(true))) + .thenReturn(Futures.immediateFuture(null)); + when(mockMobileDataDownloadManager.getDataFileUris( + FILE_GROUP_INTERNAL_1, /* verifyIsolatedStructure= */ true)) + .thenReturn( + Futures.immediateFuture( + ImmutableMap.of(FILE_GROUP_INTERNAL_1.getFile(0), onDeviceUri1))); MobileDataDownload mobileDataDownload = new MobileDataDownloadImpl( context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -2311,9 +2728,10 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); - CountDownLatch onCompleteLatch = new CountDownLatch(1); + AtomicBoolean onCompleteInvoked = new AtomicBoolean(); Account account = AccountUtil.create("account-name", "account-type"); ClientFileGroup clientFileGroup = @@ -2330,25 +2748,19 @@ public class MobileDataDownloadTest { @Override public void onComplete(ClientFileGroup clientFileGroup) { + onCompleteInvoked.set(true); assertThat(clientFileGroup.getGroupName()) .isEqualTo(FILE_GROUP_NAME_1); assertThat(clientFileGroup.getOwnerPackage()) .isEqualTo(context.getPackageName()); assertThat(clientFileGroup.getVersionNumber()).isEqualTo(5); assertThat(clientFileGroup.getFileCount()).isEqualTo(1); - - // This is to verify that onComplete is called. - onCompleteLatch.countDown(); } })) .build()) .get(); - // Verify that onComplete is called. - if (!onCompleteLatch.await(LATCH_WAIT_TIME_MS, TimeUnit.MILLISECONDS)) { - throw new RuntimeException("onComplete is not called"); - } - + assertThat(onCompleteInvoked.get()).isTrue(); assertThat(clientFileGroup.getGroupName()).isEqualTo(FILE_GROUP_NAME_1); assertThat(clientFileGroup.getOwnerPackage()).isEqualTo(context.getPackageName()); assertThat(clientFileGroup.getVersionNumber()).isEqualTo(5); @@ -2371,31 +2783,26 @@ public class MobileDataDownloadTest { @Test public void downloadFileGroup_withVariantId() throws Exception { DataFileGroupInternal dataFileGroup = - createDataFileGroupInternal( - FILE_GROUP_NAME_1, - context.getPackageName(), - /* versionNumber = */ 5, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI) - .toBuilder() - .setVariantId("en") - .build(); + FILE_GROUP_INTERNAL_1.toBuilder().setVariantId("en").build(); ArgumentCaptor<GroupKey> groupKeyCaptor = ArgumentCaptor.forClass(GroupKey.class); when(mockMobileDataDownloadManager.downloadFileGroup(groupKeyCaptor.capture(), any(), any())) .thenReturn(Futures.immediateFuture(dataFileGroup)); - when(mockMobileDataDownloadManager.getDataFileUri(dataFileGroup.getFile(0), dataFileGroup)) - .thenReturn(Futures.immediateFuture(onDeviceUri1)); + when(mockMobileDataDownloadManager.getFileGroup(any(), eq(false))) + .thenReturn(Futures.immediateFuture(FILE_GROUP_INTERNAL_1)); + when(mockMobileDataDownloadManager.getFileGroup(any(), eq(true))) + .thenReturn(Futures.immediateFuture(null)); + when(mockMobileDataDownloadManager.getDataFileUris( + dataFileGroup, /* verifyIsolatedStructure= */ true)) + .thenReturn( + Futures.immediateFuture(ImmutableMap.of(dataFileGroup.getFile(0), onDeviceUri1))); MobileDataDownload mobileDataDownload = new MobileDataDownloadImpl( context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -2403,7 +2810,8 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); ClientFileGroup clientFileGroup = mobileDataDownload @@ -2430,38 +2838,32 @@ public class MobileDataDownloadTest { @Test public void downloadFileGroupWithForegroundService() throws Exception { - DataFileGroupInternal dataFileGroup = - createDataFileGroupInternal( - FILE_GROUP_NAME_1, - context.getPackageName(), - /* versionNumber = */ 5, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI); ArgumentCaptor<GroupKey> groupKeyCaptor = ArgumentCaptor.forClass(GroupKey.class); when(mockMobileDataDownloadManager.downloadFileGroup(groupKeyCaptor.capture(), any(), any())) - .thenReturn(Futures.immediateFuture(dataFileGroup)); - when(mockMobileDataDownloadManager.getDataFileUri(dataFileGroup.getFile(0), dataFileGroup)) - .thenReturn(Futures.immediateFuture(onDeviceUri1)); + .thenReturn(Futures.immediateFuture(FILE_GROUP_INTERNAL_1)); + when(mockMobileDataDownloadManager.getDataFileUris( + FILE_GROUP_INTERNAL_1, /* verifyIsolatedStructure= */ true)) + .thenReturn( + Futures.immediateFuture( + ImmutableMap.of(FILE_GROUP_INTERNAL_1.getFile(0), onDeviceUri1))); when(mockMobileDataDownloadManager.getFileGroup(groupKeyCaptor.capture(), anyBoolean())) - .thenReturn(Futures.immediateFuture(dataFileGroup)); + .thenReturn(Futures.immediateFuture(FILE_GROUP_INTERNAL_1)); MobileDataDownload mobileDataDownload = new MobileDataDownloadImpl( context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, - /* fileGroupPopulatorList = */ ImmutableList.of(), + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), Optional.of(mockTaskScheduler), fileStorage, Optional.of(mockDownloadMonitor), Optional.of(this.getClass()), flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); CountDownLatch onCompleteLatch = new CountDownLatch(1); @@ -2518,16 +2920,6 @@ public class MobileDataDownloadTest { @Test public void downloadFileGroupWithForegroundService_failed() throws Exception { - DataFileGroupInternal dataFileGroup = - createDataFileGroupInternal( - FILE_GROUP_NAME_1, - context.getPackageName(), - /* versionNumber = */ 5, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI); ArgumentCaptor<GroupKey> groupKeyCaptor = ArgumentCaptor.forClass(GroupKey.class); when(mockMobileDataDownloadManager.downloadFileGroup(groupKeyCaptor.capture(), any(), any())) .thenReturn( @@ -2537,7 +2929,7 @@ public class MobileDataDownloadTest { .setMessage("Fail to download file group") .build())); when(mockMobileDataDownloadManager.getFileGroup(groupKeyCaptor.capture(), eq(false))) - .thenReturn(Futures.immediateFuture(dataFileGroup)); + .thenReturn(Futures.immediateFuture(FILE_GROUP_INTERNAL_1)); when(mockMobileDataDownloadManager.getFileGroup(groupKeyCaptor.capture(), eq(true))) .thenReturn(Futures.immediateFuture(null)); @@ -2546,15 +2938,21 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, - /* fileGroupPopulatorList = */ ImmutableList.of(), + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), Optional.of(mockTaskScheduler), fileStorage, Optional.of(mockDownloadMonitor), Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); + + expectErrorLogMessage( + "DownloadListener: onFailure:" + + " com.google.android.libraries.mobiledatadownload.DownloadException: Fail to download" + + " file group"); ListenableFuture<ClientFileGroup> downloadFuture = mobileDataDownload.downloadFileGroupWithForegroundService( @@ -2571,8 +2969,7 @@ public class MobileDataDownloadTest { })) .build()); - CountDownLatch onFailureLatch = new CountDownLatch(1); - + AtomicBoolean onFailureInvoked = new AtomicBoolean(); Futures.addCallback( downloadFuture, new FutureCallback<ClientFileGroup>() { @@ -2581,8 +2978,7 @@ public class MobileDataDownloadTest { @Override public void onFailure(Throwable t) { - // This is to ensure that onFailure is called. - onFailureLatch.countDown(); + onFailureInvoked.set(true); } }, MoreExecutors.directExecutor()); @@ -2591,16 +2987,11 @@ public class MobileDataDownloadTest { DownloadException e = LabsFutures.getFailureCauseAs(downloadFuture, DownloadException.class); assertThat(e).hasMessageThat().contains("Fail"); - if (!onFailureLatch.await(LATCH_WAIT_TIME_MS, TimeUnit.MILLISECONDS)) { - throw new RuntimeException("latch timeout: onFailure is not called"); - } - verify(mockMobileDataDownloadManager).downloadFileGroup(any(GroupKey.class), any(), any()); verify(mockDownloadMonitor) .addDownloadListener(eq(FILE_GROUP_NAME_1), any(DownloadListener.class)); - // Sleep for 1 sec to wait for the listener.onFailure to finish. - Thread.sleep(/*millis=*/ 1000); + assertThat(onFailureInvoked.get()).isTrue(); verify(mockDownloadMonitor).removeDownloadListener(eq(FILE_GROUP_NAME_1)); assertThat(groupKeyCaptor.getValue().getGroupName()).isEqualTo(FILE_GROUP_NAME_1); @@ -2609,23 +3000,16 @@ public class MobileDataDownloadTest { @Test public void downloadFileGroupWithForegroundService_withAccount() throws Exception { - DataFileGroupInternal dataFileGroup = - createDataFileGroupInternal( - FILE_GROUP_NAME_1, - context.getPackageName(), - 5 /* versionNumber */, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI); ArgumentCaptor<GroupKey> groupKeyCaptor = ArgumentCaptor.forClass(GroupKey.class); when(mockMobileDataDownloadManager.downloadFileGroup(groupKeyCaptor.capture(), any(), any())) - .thenReturn(Futures.immediateFuture(dataFileGroup)); - when(mockMobileDataDownloadManager.getDataFileUri(dataFileGroup.getFile(0), dataFileGroup)) - .thenReturn(Futures.immediateFuture(onDeviceUri1)); + .thenReturn(Futures.immediateFuture(FILE_GROUP_INTERNAL_1)); + when(mockMobileDataDownloadManager.getDataFileUris( + FILE_GROUP_INTERNAL_1, /* verifyIsolatedStructure= */ true)) + .thenReturn( + Futures.immediateFuture( + ImmutableMap.of(FILE_GROUP_INTERNAL_1.getFile(0), onDeviceUri1))); when(mockMobileDataDownloadManager.getFileGroup(groupKeyCaptor.capture(), eq(false))) - .thenReturn(Futures.immediateFuture(dataFileGroup)); + .thenReturn(Futures.immediateFuture(FILE_GROUP_INTERNAL_1)); when(mockMobileDataDownloadManager.getFileGroup(groupKeyCaptor.capture(), eq(true))) .thenReturn(Futures.immediateFuture(null)); @@ -2634,18 +3018,18 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, - /* fileGroupPopulatorList = */ ImmutableList.of(), + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), Optional.of(mockTaskScheduler), fileStorage, Optional.of(mockDownloadMonitor), Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); - - CountDownLatch onCompleteLatch = new CountDownLatch(1); + Optional.absent() /* customFileGroupValidator */, + timeSource); + AtomicBoolean onCompleteInvoked = new AtomicBoolean(); Account account = AccountUtil.create("account-name", "account-type"); ClientFileGroup clientFileGroup = mobileDataDownload @@ -2661,25 +3045,19 @@ public class MobileDataDownloadTest { @Override public void onComplete(ClientFileGroup clientFileGroup) { + onCompleteInvoked.set(true); assertThat(clientFileGroup.getGroupName()) .isEqualTo(FILE_GROUP_NAME_1); assertThat(clientFileGroup.getOwnerPackage()) .isEqualTo(context.getPackageName()); assertThat(clientFileGroup.getVersionNumber()).isEqualTo(5); assertThat(clientFileGroup.getFileCount()).isEqualTo(1); - - // This is to verify that onComplete is called. - onCompleteLatch.countDown(); } })) .build()) .get(); - // Verify that onComplete is called. - if (!onCompleteLatch.await(LATCH_WAIT_TIME_MS, TimeUnit.MILLISECONDS)) { - throw new RuntimeException("onComplete is not called"); - } - + assertThat(onCompleteInvoked.get()).isTrue(); assertThat(clientFileGroup.getGroupName()).isEqualTo(FILE_GROUP_NAME_1); assertThat(clientFileGroup.getOwnerPackage()).isEqualTo(context.getPackageName()); assertThat(clientFileGroup.getVersionNumber()).isEqualTo(5); @@ -2702,43 +3080,41 @@ public class MobileDataDownloadTest { @Test public void downloadFileGroupWithForegroundService_withVariantId() throws Exception { DataFileGroupInternal dataFileGroup = - createDataFileGroupInternal( - FILE_GROUP_NAME_1, - context.getPackageName(), - /* versionNumber = */ 5, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI) - .toBuilder() - .setVariantId("en") - .build(); + FILE_GROUP_INTERNAL_1.toBuilder().setVariantId("en").build(); + ArgumentCaptor<GroupKey> pendingGroupKeyCaptor = ArgumentCaptor.forClass(GroupKey.class); ArgumentCaptor<GroupKey> groupKeyCaptor = ArgumentCaptor.forClass(GroupKey.class); - when(mockMobileDataDownloadManager.downloadFileGroup(groupKeyCaptor.capture(), any(), any())) - .thenReturn(Futures.immediateFuture(dataFileGroup)); - when(mockMobileDataDownloadManager.getDataFileUri(dataFileGroup.getFile(0), dataFileGroup)) - .thenReturn(Futures.immediateFuture(onDeviceUri1)); - when(mockMobileDataDownloadManager.getFileGroup(groupKeyCaptor.capture(), eq(false))) + ArgumentCaptor<GroupKey> downloadGroupKeyCaptor = ArgumentCaptor.forClass(GroupKey.class); + when(mockMobileDataDownloadManager.downloadFileGroup( + downloadGroupKeyCaptor.capture(), any(), any())) .thenReturn(Futures.immediateFuture(dataFileGroup)); + when(mockMobileDataDownloadManager.getDataFileUris( + dataFileGroup, /* verifyIsolatedStructure= */ true)) + .thenReturn( + Futures.immediateFuture(ImmutableMap.of(dataFileGroup.getFile(0), onDeviceUri1))); + // The order here is important: first mock true and then false. + // eq(true) returns false, and so if you mock false first, pendingGroupKeyCapture will capture + // the value of groupKeyCaptor.capture(), which is null. when(mockMobileDataDownloadManager.getFileGroup(groupKeyCaptor.capture(), eq(true))) .thenReturn(Futures.immediateFuture(null)); + when(mockMobileDataDownloadManager.getFileGroup(pendingGroupKeyCaptor.capture(), eq(false))) + .thenReturn(Futures.immediateFuture(dataFileGroup)); MobileDataDownload mobileDataDownload = new MobileDataDownloadImpl( context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, - /* fileGroupPopulatorList = */ ImmutableList.of(), + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), Optional.of(mockTaskScheduler), fileStorage, Optional.of(mockDownloadMonitor), Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); ClientFileGroup clientFileGroup = mobileDataDownload @@ -2760,47 +3136,45 @@ public class MobileDataDownloadTest { .setOwnerPackage(context.getPackageName()) .setVariantId("en") .build(); + assertThat(groupKeyCaptor.getAllValues()).hasSize(1); assertThat(groupKeyCaptor.getValue()).isEqualTo(expectedGroupKey); + assertThat(pendingGroupKeyCaptor.getAllValues()).hasSize(1); + assertThat(pendingGroupKeyCaptor.getValue()).isEqualTo(expectedGroupKey); + assertThat(downloadGroupKeyCaptor.getAllValues()).hasSize(1); + assertThat(downloadGroupKeyCaptor.getValue()).isEqualTo(expectedGroupKey); } @Test public void downloadFileGroupWithForegroundService_whenAlreadyDownloaded() throws Exception { - DataFileGroupInternal dataFileGroup = - createDataFileGroupInternal( - FILE_GROUP_NAME_1, - context.getPackageName(), - /* versionNumber = */ 5, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI); - when(mockMobileDataDownloadManager.getDataFileUri(dataFileGroup.getFile(0), dataFileGroup)) - .thenReturn(Futures.immediateFuture(onDeviceUri1)); + when(mockMobileDataDownloadManager.getDataFileUris( + FILE_GROUP_INTERNAL_1, /* verifyIsolatedStructure= */ true)) + .thenReturn( + Futures.immediateFuture( + ImmutableMap.of(FILE_GROUP_INTERNAL_1.getFile(0), onDeviceUri1))); // Mock situation: no pending group but there is a downloaded group when(mockMobileDataDownloadManager.getFileGroup(any(), eq(false))) .thenReturn(Futures.immediateFuture(null)); when(mockMobileDataDownloadManager.getFileGroup(any(), eq(true))) - .thenReturn(Futures.immediateFuture(dataFileGroup)); + .thenReturn(Futures.immediateFuture(FILE_GROUP_INTERNAL_1)); MobileDataDownload mobileDataDownload = new MobileDataDownloadImpl( context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, - /* fileGroupPopulatorList = */ ImmutableList.of(), + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), Optional.of(mockTaskScheduler), fileStorage, Optional.of(mockDownloadMonitor), Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); - - CountDownLatch onCompleteLatch = new CountDownLatch(1); + Optional.absent() /* customFileGroupValidator */, + timeSource); + AtomicBoolean onCompleteInvoked = new AtomicBoolean(false); ClientFileGroup clientFileGroup = mobileDataDownload .downloadFileGroupWithForegroundService( @@ -2814,25 +3188,19 @@ public class MobileDataDownloadTest { @Override public void onComplete(ClientFileGroup clientFileGroup) { + onCompleteInvoked.set(true); assertThat(clientFileGroup.getGroupName()) .isEqualTo(FILE_GROUP_NAME_1); assertThat(clientFileGroup.getOwnerPackage()) .isEqualTo(context.getPackageName()); assertThat(clientFileGroup.getVersionNumber()).isEqualTo(5); assertThat(clientFileGroup.getFileCount()).isEqualTo(1); - - // This is to verify that onComplete is called. - onCompleteLatch.countDown(); } })) .build()) .get(); - // Verify that onComplete is called. - if (!onCompleteLatch.await(LATCH_WAIT_TIME_MS, TimeUnit.MILLISECONDS)) { - throw new RuntimeException("onComplete is not called"); - } - + assertThat(onCompleteInvoked.get()).isTrue(); assertThat(clientFileGroup.getGroupName()).isEqualTo(FILE_GROUP_NAME_1); assertThat(clientFileGroup.getOwnerPackage()).isEqualTo(context.getPackageName()); assertThat(clientFileGroup.getVersionNumber()).isEqualTo(5); @@ -2862,18 +3230,18 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, - /* fileGroupPopulatorList = */ ImmutableList.of(), + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), Optional.of(mockTaskScheduler), fileStorage, Optional.of(mockDownloadMonitor), Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); - - CountDownLatch onFailureLatch = new CountDownLatch(1); + Optional.absent() /* customFileGroupValidator */, + timeSource); + AtomicBoolean onFailureInvoked = new AtomicBoolean(false); ListenableFuture<ClientFileGroup> downloadFuture = mobileDataDownload.downloadFileGroupWithForegroundService( DownloadFileGroupRequest.newBuilder() @@ -2891,23 +3259,17 @@ public class MobileDataDownloadTest { @Override public void onFailure(Throwable t) { + onFailureInvoked.set(true); assertThat(t).isInstanceOf(DownloadException.class); assertThat(((DownloadException) t).getDownloadResultCode()) .isEqualTo(DownloadResultCode.GROUP_NOT_FOUND_ERROR); - - // This is to verify onFailure is called. - onFailureLatch.countDown(); } })) .build()); assertThrows(ExecutionException.class, downloadFuture::get); - // Verify onFailure is called - if (!onFailureLatch.await(LATCH_WAIT_TIME_MS, TimeUnit.MILLISECONDS)) { - fail("onFailure should be called"); - } - + assertThat(onFailureInvoked.get()).isTrue(); DownloadException e = LabsFutures.getFailureCauseAs(downloadFuture, DownloadException.class); assertThat(e.getDownloadResultCode()).isEqualTo(DownloadResultCode.GROUP_NOT_FOUND_ERROR); @@ -2925,15 +3287,16 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, - /*fileGroupPopulatorList=*/ ImmutableList.of(), + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), Optional.of(mockTaskScheduler), fileStorage, - /*downloadMonitorOptional=*/ Optional.absent(), + /* downloadMonitorOptional= */ Optional.absent(), Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); when(mockMobileDataDownloadManager.maintenance()).thenReturn(Futures.immediateFuture(null)); @@ -2950,15 +3313,16 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, - /*fileGroupPopulatorList=*/ ImmutableList.of(), + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), Optional.of(mockTaskScheduler), fileStorage, - /*downloadMonitorOptional=*/ Optional.absent(), + /* downloadMonitorOptional= */ Optional.absent(), Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); when(mockMobileDataDownloadManager.maintenance()) .thenReturn(Futures.immediateFailedFuture(new IOException("test-failure"))); @@ -2973,51 +3337,79 @@ public class MobileDataDownloadTest { } @Test + public void collectGarbage_interactionTest() throws Exception { + MobileDataDownload mobileDataDownload = + new MobileDataDownloadImpl( + context, + mockEventLogger, + mockMobileDataDownloadManager, + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), + Optional.of(mockTaskScheduler), + fileStorage, + /* downloadMonitorOptional= */ Optional.absent(), + Optional.of(this.getClass()), // don't need to use the real foreground download service. + flags, + singleFileDownloader, + Optional.absent() /* customFileGroupValidator */, + timeSource); + + when(mockMobileDataDownloadManager.removeExpiredGroupsAndFiles()) + .thenReturn(Futures.immediateFuture(null)); + + mobileDataDownload.collectGarbage().get(); + + verify(mockMobileDataDownloadManager).removeExpiredGroupsAndFiles(); + verifyNoMoreInteractions(mockMobileDataDownloadManager); + } + + @Test public void schedulePeriodicTasks() throws Exception { MobileDataDownload mobileDataDownload = new MobileDataDownloadImpl( context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, - /* fileGroupPopulatorList = */ ImmutableList.of(), + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), Optional.of(mockTaskScheduler), fileStorage, - /* downloadMonitorOptional = */ Optional.absent(), + /* downloadMonitorOptional= */ Optional.absent(), Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); mobileDataDownload.schedulePeriodicTasks(); verify(mockTaskScheduler) .schedulePeriodicTask( TaskScheduler.CHARGING_PERIODIC_TASK, - (new Flags() {}).chargingGcmTaskPeriod(), + flags.chargingGcmTaskPeriod(), NetworkState.NETWORK_STATE_ANY, - /* constraintOverrides = */ Optional.absent()); + /* constraintOverrides= */ Optional.absent()); verify(mockTaskScheduler) .schedulePeriodicTask( TaskScheduler.MAINTENANCE_PERIODIC_TASK, - (new Flags() {}).maintenanceGcmTaskPeriod(), + flags.maintenanceGcmTaskPeriod(), NetworkState.NETWORK_STATE_ANY, - /* constraintOverrides = */ Optional.absent()); + /* constraintOverrides= */ Optional.absent()); verify(mockTaskScheduler) .schedulePeriodicTask( TaskScheduler.CELLULAR_CHARGING_PERIODIC_TASK, - (new Flags() {}).cellularChargingGcmTaskPeriod(), + flags.cellularChargingGcmTaskPeriod(), NetworkState.NETWORK_STATE_CONNECTED, - /* constraintOverrides = */ Optional.absent()); + /* constraintOverrides= */ Optional.absent()); verify(mockTaskScheduler) .schedulePeriodicTask( TaskScheduler.WIFI_CHARGING_PERIODIC_TASK, - (new Flags() {}).wifiChargingGcmTaskPeriod(), + flags.wifiChargingGcmTaskPeriod(), NetworkState.NETWORK_STATE_UNMETERED, - /* constraintOverrides = */ Optional.absent()); + /* constraintOverrides= */ Optional.absent()); verifyNoMoreInteractions(mockTaskScheduler); } @@ -3029,16 +3421,20 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, - /* fileGroupPopulatorList = */ ImmutableList.of(), - /* taskSchedulerOptional = */ Optional.absent(), + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), + /* taskSchedulerOptional= */ Optional.absent(), fileStorage, - /* downloadMonitorOptional = */ Optional.absent(), + /* downloadMonitorOptional= */ Optional.absent(), Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); + expectErrorLogMessage( + "MobileDataDownload: Called schedulePeriodicTasksInternal when taskScheduler is not" + + " provided."); mobileDataDownload.schedulePeriodicTasks(); verifyNoInteractions(mockTaskScheduler); @@ -3051,45 +3447,46 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, - /* fileGroupPopulatorList = */ ImmutableList.of(), + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), Optional.of(mockTaskScheduler), fileStorage, - /* downloadMonitorOptional = */ Optional.absent(), + /* downloadMonitorOptional= */ Optional.absent(), Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); mobileDataDownload.schedulePeriodicBackgroundTasks().get(); verify(mockTaskScheduler) .schedulePeriodicTask( TaskScheduler.CHARGING_PERIODIC_TASK, - (new Flags() {}).chargingGcmTaskPeriod(), + flags.chargingGcmTaskPeriod(), NetworkState.NETWORK_STATE_ANY, - /* constraintOverrides = */ Optional.absent()); + /* constraintOverrides= */ Optional.absent()); verify(mockTaskScheduler) .schedulePeriodicTask( TaskScheduler.MAINTENANCE_PERIODIC_TASK, - (new Flags() {}).maintenanceGcmTaskPeriod(), + flags.maintenanceGcmTaskPeriod(), NetworkState.NETWORK_STATE_ANY, - /* constraintOverrides = */ Optional.absent()); + /* constraintOverrides= */ Optional.absent()); verify(mockTaskScheduler) .schedulePeriodicTask( TaskScheduler.CELLULAR_CHARGING_PERIODIC_TASK, - (new Flags() {}).cellularChargingGcmTaskPeriod(), + flags.cellularChargingGcmTaskPeriod(), NetworkState.NETWORK_STATE_CONNECTED, - /* constraintOverrides = */ Optional.absent()); + /* constraintOverrides= */ Optional.absent()); verify(mockTaskScheduler) .schedulePeriodicTask( TaskScheduler.WIFI_CHARGING_PERIODIC_TASK, - (new Flags() {}).wifiChargingGcmTaskPeriod(), + flags.wifiChargingGcmTaskPeriod(), NetworkState.NETWORK_STATE_UNMETERED, - /* constraintOverrides = */ Optional.absent()); + /* constraintOverrides= */ Optional.absent()); verifyNoMoreInteractions(mockTaskScheduler); } @@ -3101,16 +3498,20 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, - /* fileGroupPopulatorList = */ ImmutableList.of(), - /* taskSchedulerOptional = */ Optional.absent(), + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), + /* taskSchedulerOptional= */ Optional.absent(), fileStorage, - /* downloadMonitorOptional = */ Optional.absent(), + /* downloadMonitorOptional= */ Optional.absent(), Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); + expectErrorLogMessage( + "MobileDataDownload: Called schedulePeriodicTasksInternal when taskScheduler is not" + + " provided."); mobileDataDownload.schedulePeriodicBackgroundTasks().get(); verifyNoInteractions(mockTaskScheduler); @@ -3123,15 +3524,16 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, - /* fileGroupPopulatorList = */ ImmutableList.of(), + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), Optional.of(mockTaskScheduler), fileStorage, - /* downloadMonitorOptional = */ Optional.absent(), + /* downloadMonitorOptional= */ Optional.absent(), Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); ConstraintOverrides wifiOverrides = ConstraintOverrides.newBuilder() @@ -3153,28 +3555,28 @@ public class MobileDataDownloadTest { verify(mockTaskScheduler) .schedulePeriodicTask( TaskScheduler.CHARGING_PERIODIC_TASK, - (new Flags() {}).chargingGcmTaskPeriod(), + flags.chargingGcmTaskPeriod(), NetworkState.NETWORK_STATE_ANY, Optional.absent()); verify(mockTaskScheduler) .schedulePeriodicTask( TaskScheduler.MAINTENANCE_PERIODIC_TASK, - (new Flags() {}).maintenanceGcmTaskPeriod(), + flags.maintenanceGcmTaskPeriod(), NetworkState.NETWORK_STATE_ANY, Optional.absent()); verify(mockTaskScheduler) .schedulePeriodicTask( TaskScheduler.CELLULAR_CHARGING_PERIODIC_TASK, - (new Flags() {}).cellularChargingGcmTaskPeriod(), + flags.cellularChargingGcmTaskPeriod(), NetworkState.NETWORK_STATE_CONNECTED, Optional.of(cellularOverrides)); verify(mockTaskScheduler) .schedulePeriodicTask( TaskScheduler.WIFI_CHARGING_PERIODIC_TASK, - (new Flags() {}).wifiChargingGcmTaskPeriod(), + flags.wifiChargingGcmTaskPeriod(), NetworkState.NETWORK_STATE_UNMETERED, Optional.of(wifiOverrides)); @@ -3188,21 +3590,80 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, - /* fileGroupPopulatorList = */ ImmutableList.of(), - /* taskSchedulerOptional = */ Optional.absent(), + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), + /* taskSchedulerOptional= */ Optional.absent(), fileStorage, - /* downloadMonitorOptional = */ Optional.absent(), + /* downloadMonitorOptional= */ Optional.absent(), Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); + + expectErrorLogMessage( + "MobileDataDownload: Called schedulePeriodicTasksInternal when taskScheduler is not" + + " provided."); mobileDataDownload.schedulePeriodicBackgroundTasks(Optional.absent()).get(); verifyNoInteractions(mockTaskScheduler); } + @Test + public void cancelPeriodicBackgroundTasks_nullTaskScheduler() throws Exception { + MobileDataDownload mobileDataDownload = + new MobileDataDownloadImpl( + context, + mockEventLogger, + mockMobileDataDownloadManager, + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), + /* taskSchedulerOptional= */ Optional.absent(), + fileStorage, + /* downloadMonitorOptional= */ Optional.absent(), + Optional.absent() /* foregroundDownloadServiceClassOptional */, + flags, + singleFileDownloader, + Optional.absent() /* customFileGroupValidator */, + timeSource); + + mobileDataDownload.cancelPeriodicBackgroundTasks().get(); + + verifyNoInteractions(mockTaskScheduler); + } + + @Test + public void cancelPeriodicBackgroundTasks() throws Exception { + MobileDataDownload mobileDataDownload = + new MobileDataDownloadImpl( + context, + mockEventLogger, + mockMobileDataDownloadManager, + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), + Optional.of(mockTaskScheduler), + fileStorage, + /* downloadMonitorOptional= */ Optional.absent(), + Optional.absent() /* foregroundDownloadServiceClassOptional */, + flags, + singleFileDownloader, + Optional.absent() /* customFileGroupValidator */, + timeSource); + + mobileDataDownload.cancelPeriodicBackgroundTasks().get(); + + verify(mockTaskScheduler).cancelPeriodicTask(TaskScheduler.CHARGING_PERIODIC_TASK); + + verify(mockTaskScheduler).cancelPeriodicTask(TaskScheduler.MAINTENANCE_PERIODIC_TASK); + + verify(mockTaskScheduler).cancelPeriodicTask(TaskScheduler.CELLULAR_CHARGING_PERIODIC_TASK); + + verify(mockTaskScheduler).cancelPeriodicTask(TaskScheduler.WIFI_CHARGING_PERIODIC_TASK); + + verifyNoMoreInteractions(mockTaskScheduler); + } + // A helper function to create a DataFilegroup. private static DataFileGroup createDataFileGroup( String groupName, @@ -3249,18 +3710,22 @@ public class MobileDataDownloadTest { int[] byteSize, String[] checksum, String[] url, - DeviceNetworkPolicy deviceNetworkPolicy) - throws Exception { - return ProtoConversionUtil.convert( - createDataFileGroup( - groupName, - ownerPackage, - versionNumber, - fileId, - byteSize, - checksum, - url, - deviceNetworkPolicy)); + DeviceNetworkPolicy deviceNetworkPolicy) { + try { + return ProtoConversionUtil.convert( + createDataFileGroup( + groupName, + ownerPackage, + versionNumber, + fileId, + byteSize, + checksum, + url, + deviceNetworkPolicy)); + } catch (Exception e) { + // wrap with runtime exception to avoid this method having to declare throws + throw new RuntimeException(e); + } } @Test @@ -3270,15 +3735,16 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, - /* fileGroupPopulatorList = */ ImmutableList.of(), + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), Optional.of(mockTaskScheduler), fileStorage, - /* downloadMonitorOptional = */ Optional.absent(), + /* downloadMonitorOptional= */ Optional.absent(), Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); when(mockMobileDataDownloadManager.maintenance()).thenReturn(Futures.immediateFuture(null)); mobileDataDownload.handleTask(TaskScheduler.MAINTENANCE_PERIODIC_TASK).get(); @@ -3293,15 +3759,16 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of(mockFileGroupPopulator), Optional.of(mockTaskScheduler), fileStorage, - /* downloadMonitorOptional = */ Optional.absent(), + /* downloadMonitorOptional= */ Optional.absent(), Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); when(mockMobileDataDownloadManager.verifyAllPendingGroups(any())) .thenReturn(Futures.immediateFuture(null)); @@ -3321,15 +3788,16 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of(mockFileGroupPopulator), Optional.of(mockTaskScheduler), fileStorage, - /* downloadMonitorOptional = */ Optional.absent(), + /* downloadMonitorOptional= */ Optional.absent(), Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); when(mockFileGroupPopulator.refreshFileGroups(mobileDataDownload)) .thenReturn(Futures.immediateFuture(null)); @@ -3349,15 +3817,16 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of(mockFileGroupPopulator), Optional.of(mockTaskScheduler), fileStorage, - /* downloadMonitorOptional = */ Optional.absent(), + /* downloadMonitorOptional= */ Optional.absent(), Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); when(mockFileGroupPopulator.refreshFileGroups(mobileDataDownload)) .thenReturn(Futures.immediateFuture(null)); @@ -3377,15 +3846,16 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, - /* fileGroupPopulatorList = */ ImmutableList.of(), + controlExecutor, + /* fileGroupPopulatorList= */ ImmutableList.of(), Optional.of(mockTaskScheduler), fileStorage, - /* downloadMonitorOptional = */ Optional.absent(), + /* downloadMonitorOptional= */ Optional.absent(), Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); when(mockMobileDataDownloadManager.verifyAllPendingGroups(any())) .thenReturn(Futures.immediateFuture(null)); @@ -3409,15 +3879,16 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of(mockFileGroupPopulator), Optional.of(mockTaskScheduler), fileStorage, - /* downloadMonitorOptional = */ Optional.absent(), + /* downloadMonitorOptional= */ Optional.absent(), Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); ExecutionException e = assertThrows( @@ -3454,15 +3925,16 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, populators, Optional.of(mockTaskScheduler), fileStorage, - /* downloadMonitorOptional = */ Optional.absent(), + /* downloadMonitorOptional= */ Optional.absent(), Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); + Optional.absent() /* customFileGroupValidator */, + timeSource); when(mockMobileDataDownloadManager.verifyAllPendingGroups(any() /* validator */)) .thenReturn(Futures.immediateVoidFuture()); @@ -3481,12 +3953,13 @@ public class MobileDataDownloadTest { @Test public void reportUsage_basic() throws Exception { - DataFileGroupInternal dataFileGroup = createDefaultDataFileGroupInternal(); - when(mockMobileDataDownloadManager.getFileGroup(any(GroupKey.class), eq(true))) - .thenReturn(Futures.immediateFuture(dataFileGroup)); - when(mockMobileDataDownloadManager.getDataFileUri(dataFileGroup.getFile(0), dataFileGroup)) - .thenReturn(Futures.immediateFuture(onDeviceUri1)); + .thenReturn(Futures.immediateFuture(FILE_GROUP_INTERNAL_1)); + when(mockMobileDataDownloadManager.getDataFileUris( + FILE_GROUP_INTERNAL_1, /* verifyIsolatedStructure= */ true)) + .thenReturn( + Futures.immediateFuture( + ImmutableMap.of(FILE_GROUP_INTERNAL_1.getFile(0), onDeviceUri1))); MobileDataDownload mobileDataDownload = createDefaultMobileDataDownload(); @@ -3506,8 +3979,15 @@ public class MobileDataDownloadTest { verify(mockEventLogger).logMddUsageEvent(createFileGroupStats(clientFileGroup), null); } - private static Void createFileGroupStats(ClientFileGroup clientFileGroup) { - return null; + private static DataDownloadFileGroupStats createFileGroupStats(ClientFileGroup clientFileGroup) { + return DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(clientFileGroup.getGroupName()) + .setOwnerPackage(clientFileGroup.getOwnerPackage()) + .setFileGroupVersionNumber(clientFileGroup.getVersionNumber()) + .setFileCount(clientFileGroup.getFileCount()) + .setVariantId(clientFileGroup.getVariantId()) + .setBuildId(clientFileGroup.getBuildId()) + .build(); } private MobileDataDownload createDefaultMobileDataDownload() { @@ -3515,7 +3995,7 @@ public class MobileDataDownloadTest { context, mockEventLogger, mockMobileDataDownloadManager, - EXECUTOR, + controlExecutor, ImmutableList.of() /* fileGroupPopulatorList */, Optional.of(mockTaskScheduler), fileStorage, @@ -3523,18 +4003,7 @@ public class MobileDataDownloadTest { Optional.of(this.getClass()), // don't need to use the real foreground download service. flags, singleFileDownloader, - Optional.absent() /* customFileGroupValidator */); - } - - private DataFileGroupInternal createDefaultDataFileGroupInternal() throws Exception { - return createDataFileGroupInternal( - FILE_GROUP_NAME_1, - context.getPackageName(), - 1 /* versionNumber */, - new String[] {FILE_ID_1}, - new int[] {FILE_SIZE_1}, - new String[] {FILE_CHECKSUM_1}, - new String[] {FILE_URL_1}, - DeviceNetworkPolicy.DOWNLOAD_ONLY_ON_WIFI); + Optional.absent() /* customFileGroupValidator */, + timeSource); } } diff --git a/javatests/com/google/android/libraries/mobiledatadownload/TestFileGroupPopulator.java b/javatests/com/google/android/libraries/mobiledatadownload/TestFileGroupPopulator.java index c089e95..c6503af 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/TestFileGroupPopulator.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/TestFileGroupPopulator.java @@ -22,6 +22,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import com.google.mobiledatadownload.DownloadConfigProto.DataFile; +import com.google.mobiledatadownload.DownloadConfigProto.DataFile.ChecksumType; import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; import com.google.mobiledatadownload.DownloadConfigProto.DownloadConditions; import com.google.mobiledatadownload.DownloadConfigProto.DownloadConditions.DeviceNetworkPolicy; @@ -96,14 +97,16 @@ public class TestFileGroupPopulator implements FileGroupPopulator { DownloadConditions.newBuilder().setDeviceNetworkPolicy(deviceNetworkPolicy)); for (int i = 0; i < fileId.length; ++i) { - DataFile file = + DataFile.Builder fileBuilder = DataFile.newBuilder() .setFileId(fileId[i]) .setByteSize(byteSize[i]) .setChecksum(checksum[i]) - .setUrlToDownload(url[i]) - .build(); - dataFileGroupBuilder.addFile(file); + .setUrlToDownload(url[i]); + if (checksum[i].isEmpty()) { + fileBuilder.setChecksumType(ChecksumType.NONE); + } + dataFileGroupBuilder.addFile(fileBuilder.build()); } return dataFileGroupBuilder.build(); @@ -139,6 +142,9 @@ public class TestFileGroupPopulator implements FileGroupPopulator { .setByteSize(byteSize[i]) .setChecksum(checksum[i]) .setUrlToDownload(url[i]); + if (checksum[i].isEmpty()) { + fileBuilder.setChecksumType(ChecksumType.NONE); + } if (!TextUtils.isEmpty(androidSharingChecksum[i])) { fileBuilder .setAndroidSharingType(DataFile.AndroidSharingType.ANDROID_BLOB_WHEN_AVAILABLE) diff --git a/javatests/com/google/android/libraries/mobiledatadownload/TwoStepPopulator.java b/javatests/com/google/android/libraries/mobiledatadownload/TwoStepPopulator.java index 881c4a4..16f5357 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/TwoStepPopulator.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/TwoStepPopulator.java @@ -30,7 +30,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import com.google.mobiledatadownload.ClientConfigProto.ClientFileGroup; -import com.google.mobiledatadownload.DownloadConfigProto.DataFile.ChecksumType; import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; import com.google.mobiledatadownload.DownloadConfigProto.DownloadConditions.DeviceNetworkPolicy; import java.io.IOException; @@ -76,13 +75,12 @@ public class TwoStepPopulator implements FileGroupPopulator { // Add a file group where the url is read from step1.txt DataFileGroup step2FileGroup = - MobileDataDownloadIntegrationTest.createDataFileGroup( + TestFileGroupPopulator.createDataFileGroup( "step2-file-group", context.getPackageName(), new String[] {"step2_id"}, new int[] {13}, new String[] {""}, - new ChecksumType[] {ChecksumType.NONE}, new String[] {step1Content}, DeviceNetworkPolicy.DOWNLOAD_ON_ANY_NETWORK); diff --git a/javatests/com/google/android/libraries/mobiledatadownload/account/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/account/BUILD index 48e23d9..9263af0 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/account/BUILD +++ b/javatests/com/google/android/libraries/mobiledatadownload/account/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_local_test") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -27,6 +28,7 @@ android_local_test( }, deps = [ "//java/com/google/android/libraries/mobiledatadownload/account:AccountUtil", + "//java/com/google/android/libraries/mobiledatadownload/internal:MddConstants", "@truth", ], ) diff --git a/javatests/com/google/android/libraries/mobiledatadownload/downloader/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/downloader/BUILD index 868a547..38d1c5d 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/downloader/BUILD +++ b/javatests/com/google/android/libraries/mobiledatadownload/downloader/BUILD @@ -14,6 +14,7 @@ load("//javatests/com/google/android/libraries/mobiledatadownload:test_defs.bzl", "mdd_local_test") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -25,8 +26,9 @@ mdd_local_test( deps = [ "//java/com/google/android/libraries/mobiledatadownload/downloader:FileDownloader", "@android_sdk_linux", - "@androidx_test", + "@androidx_concurrent_concurrent", "@com_google_guava_guava", + "@junit", "@mockito", "@truth", ], @@ -42,6 +44,7 @@ mdd_local_test( "//java/com/google/android/libraries/mobiledatadownload/internal:MddConstants", "@androidx_test", "@com_google_protobuf//:protobuf_lite", + "@junit", "@truth", ], ) diff --git a/javatests/com/google/android/libraries/mobiledatadownload/downloader/inline/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/downloader/inline/BUILD index 210f5ce..092b9d9 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/downloader/inline/BUILD +++ b/javatests/com/google/android/libraries/mobiledatadownload/downloader/inline/BUILD @@ -14,6 +14,7 @@ load("//javatests/com/google/android/libraries/mobiledatadownload:test_defs.bzl", "mdd_local_test") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -31,7 +32,6 @@ mdd_local_test( "//java/com/google/android/libraries/mobiledatadownload/file/backends:android", "//java/com/google/android/libraries/mobiledatadownload/file/common/testing:fake_file_backend", "//java/com/google/android/libraries/mobiledatadownload/file/openers:stream", - "@androidx_test", "@com_google_guava_guava", "@com_google_protobuf//:protobuf_lite", "@truth", diff --git a/javatests/com/google/android/libraries/mobiledatadownload/downloader/inline/InlineFileDownloaderTest.java b/javatests/com/google/android/libraries/mobiledatadownload/downloader/inline/InlineFileDownloaderTest.java index a7e392b..0d24114 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/downloader/inline/InlineFileDownloaderTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/downloader/inline/InlineFileDownloaderTest.java @@ -58,9 +58,9 @@ public final class InlineFileDownloaderTest { new FakeFileBackend(AndroidFileBackend.builder(CONTEXT).build()); private static final SynchronousFileStorage FILE_STORAGE = new SynchronousFileStorage( - /* backends = */ ImmutableList.of(FAKE_FILE_BACKEND), - /* transforms = */ ImmutableList.of(), - /* monitors = */ ImmutableList.of()); + /* backends= */ ImmutableList.of(FAKE_FILE_BACKEND), + /* transforms= */ ImmutableList.of(), + /* monitors= */ ImmutableList.of()); private final Uri fileUri = Uri.parse( diff --git a/javatests/com/google/android/libraries/mobiledatadownload/downloader/offroad/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/downloader/offroad/BUILD index 7fbd330..374a6a3 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/downloader/offroad/BUILD +++ b/javatests/com/google/android/libraries/mobiledatadownload/downloader/offroad/BUILD @@ -14,6 +14,7 @@ load("//javatests/com/google/android/libraries/mobiledatadownload:test_defs.bzl", "mdd_local_test") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -32,15 +33,18 @@ mdd_local_test( "//java/com/google/android/libraries/mobiledatadownload/downloader/offroad:ExceptionHandler", "//java/com/google/android/libraries/mobiledatadownload/downloader/offroad:Offroad2FileDownloader", "//java/com/google/android/libraries/mobiledatadownload/file", + "//java/com/google/android/libraries/mobiledatadownload/file/backends", "//java/com/google/android/libraries/mobiledatadownload/file/backends:android", "//java/com/google/android/libraries/mobiledatadownload/file/backends:file", "//java/com/google/android/libraries/mobiledatadownload/file/common/testing", "//java/com/google/android/libraries/mobiledatadownload/file/integration/downloader:downloader2", "//java/com/google/android/libraries/mobiledatadownload/file/openers:stream", + "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "//javatests/com/google/android/libraries/mobiledatadownload/internal:MddTestUtil", "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestHttpServer", + "//third_party/java/android_libs/downloader:contrib", "@android_sdk_linux", - "@androidx_test", "@com_google_guava_guava", "@com_google_runfiles", "@downloader", diff --git a/javatests/com/google/android/libraries/mobiledatadownload/downloader/offroad/ExceptionHandlerTest.java b/javatests/com/google/android/libraries/mobiledatadownload/downloader/offroad/ExceptionHandlerTest.java index ce6b155..ac9b268 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/downloader/offroad/ExceptionHandlerTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/downloader/offroad/ExceptionHandlerTest.java @@ -33,9 +33,9 @@ public final class ExceptionHandlerTest { public void mapToDownloadException_withDefaultImpl_handlesHttpStatusErrors() throws Exception { ErrorDetails errorDetails = ErrorDetails.createFromHttpErrorResponse( - /* httpResponseCode = */ 404, - /* httpResponseHeaders = */ ImmutableMap.of(), - /* message = */ "404 response"); + /* httpResponseCode= */ 404, + /* httpResponseHeaders= */ ImmutableMap.of(), + /* message= */ "404 response"); RequestException requestException = new RequestException(errorDetails); ExceptionHandler handler = ExceptionHandler.withDefaultHandling(); @@ -54,9 +54,9 @@ public final class ExceptionHandlerTest { throws Exception { ErrorDetails errorDetails = ErrorDetails.createFromHttpErrorResponse( - /* httpResponseCode = */ 404, - /* httpResponseHeaders = */ ImmutableMap.of(), - /* message = */ "404 response"); + /* httpResponseCode= */ 404, + /* httpResponseHeaders= */ ImmutableMap.of(), + /* message= */ "404 response"); RequestException requestException = new RequestException(errorDetails); com.google.android.downloader.DownloadException wrappedException = diff --git a/javatests/com/google/android/libraries/mobiledatadownload/downloader/offroad/Offroad2FileDownloaderTest.java b/javatests/com/google/android/libraries/mobiledatadownload/downloader/offroad/Offroad2FileDownloaderTest.java index d9f439b..355c21e 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/downloader/offroad/Offroad2FileDownloaderTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/downloader/offroad/Offroad2FileDownloaderTest.java @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -// TODO package com.google.android.libraries.mobiledatadownload.downloader.offroad; import static com.google.common.truth.Truth.assertThat; @@ -28,6 +27,7 @@ import android.net.Uri; import android.util.Pair; import androidx.test.core.app.ApplicationProvider; import com.google.android.downloader.ConnectivityHandler; +import com.google.android.downloader.CookieJar; import com.google.android.downloader.DownloadConstraints; import com.google.android.downloader.DownloadConstraints.NetworkType; import com.google.android.downloader.DownloadMetadata; @@ -35,6 +35,7 @@ import com.google.android.downloader.Downloader; import com.google.android.downloader.FloggerDownloaderLogger; import com.google.android.downloader.OAuthTokenProvider; import com.google.android.downloader.PlatformUrlEngine; +import com.google.android.downloader.contrib.InMemoryCookieJar; import com.google.android.downloader.testing.TestUrlEngine; import com.google.android.downloader.testing.TestUrlEngine.TestUrlRequest; import com.google.android.libraries.mobiledatadownload.DownloadException; @@ -118,21 +119,23 @@ public class Offroad2FileDownloaderTest { private FakeOAuthTokenProvider fakeOAuthTokenProvider; private FakeTrafficStatsTagger fakeTrafficStatsTagger; private TestUrlEngine testUrlEngine; + private CookieJar cookieJar; private Downloader downloader; private Offroad2FileDownloader fileDownloader; - @Rule public TemporaryUri tmpUri = new TemporaryUri(); + @Rule(order = 1) + public TemporaryUri tmpUri = new TemporaryUri(); @Before public void setUp() throws Exception { context = ApplicationProvider.getApplicationContext(); fileStorage = new SynchronousFileStorage( - /* backends = */ ImmutableList.of( + /* backends= */ ImmutableList.of( AndroidFileBackend.builder(context).build(), new JavaFileBackend()), - /* transforms = */ ImmutableList.of(), - /* monitors = */ ImmutableList.of()); + /* transforms= */ ImmutableList.of(), + /* monitors= */ ImmutableList.of()); fakeDownloadMetadataStore = new FakeDownloadMetadataStore(); @@ -143,6 +146,7 @@ public class Offroad2FileDownloaderTest { CONTROL_EXECUTOR, MAX_PLATFORM_ENGINE_TIMEOUT_MILLIS, MAX_PLATFORM_ENGINE_TIMEOUT_MILLIS, + /* followHttpRedirects= */ false, fakeTrafficStatsTagger); testUrlEngine = new TestUrlEngine(urlEngine); @@ -160,6 +164,8 @@ public class Offroad2FileDownloaderTest { fakeOAuthTokenProvider = new FakeOAuthTokenProvider(); + cookieJar = new InMemoryCookieJar(); + fileDownloader = new Offroad2FileDownloader( downloader, @@ -168,6 +174,7 @@ public class Offroad2FileDownloaderTest { fakeOAuthTokenProvider, fakeDownloadMetadataStore, ExceptionHandler.withDefaultHandling(), + Optional.of(() -> cookieJar), Optional.absent()); testHttpServer = new TestHttpServer(); @@ -175,7 +182,7 @@ public class Offroad2FileDownloaderTest { } @After - public void tearDown() { + public void tearDown() throws Exception { testHttpServer.stopServer(); fakeConnectivityHandler.reset(); fakeDownloadMetadataStore.reset(); diff --git a/javatests/com/google/android/libraries/mobiledatadownload/file/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/file/BUILD index d02d4f1..b2584a5 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/file/BUILD +++ b/javatests/com/google/android/libraries/mobiledatadownload/file/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_local_test") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -65,7 +66,6 @@ android_local_test( "//java/com/google/android/libraries/mobiledatadownload/file/spi", "//java/com/google/android/libraries/mobiledatadownload/file/transforms:buffer", "//java/com/google/android/libraries/mobiledatadownload/file/transforms:compress", - "@androidx_test", "@com_google_guava_guava", "@mockito", "@truth", diff --git a/javatests/com/google/android/libraries/mobiledatadownload/file/SynchronousFileStorageTest.java b/javatests/com/google/android/libraries/mobiledatadownload/file/SynchronousFileStorageTest.java index e3dc018..897e69c 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/file/SynchronousFileStorageTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/file/SynchronousFileStorageTest.java @@ -110,7 +110,7 @@ public class SynchronousFileStorageTest extends FileStorageTestBase { return ""; } }; - new SynchronousFileStorage(ImmutableList.of(emptyNameBackend)); + var unused = new SynchronousFileStorage(ImmutableList.of(emptyNameBackend)); } @Test @@ -278,7 +278,8 @@ public class SynchronousFileStorageTest extends FileStorageTestBase { return ""; } }; - new SynchronousFileStorage(ImmutableList.of(), ImmutableList.of(emptyNameTransform)); + var unused = + new SynchronousFileStorage(ImmutableList.of(), ImmutableList.of(emptyNameTransform)); } @Test diff --git a/javatests/com/google/android/libraries/mobiledatadownload/file/backends/AndroidFileBackendTest.java b/javatests/com/google/android/libraries/mobiledatadownload/file/backends/AndroidFileBackendTest.java index 61c8912..639f67f 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/file/backends/AndroidFileBackendTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/file/backends/AndroidFileBackendTest.java @@ -40,6 +40,7 @@ import com.google.android.libraries.mobiledatadownload.file.openers.NativeReadOp import com.google.android.libraries.mobiledatadownload.file.openers.ReadStreamOpener; import com.google.android.libraries.mobiledatadownload.file.spi.Backend; import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.Futures; import java.io.ByteArrayInputStream; import java.io.Closeable; import java.io.File; @@ -53,7 +54,9 @@ import org.robolectric.annotation.Config; /** Tests for {@link AndroidFileBackend} */ @RunWith(RobolectricTestRunner.class) -@Config(sdk = Build.VERSION_CODES.N) +@Config( + shadows = {}, + sdk = Build.VERSION_CODES.N) public class AndroidFileBackendTest extends BackendTestBase { private final Context context = ApplicationProvider.getApplicationContext(); @@ -261,6 +264,70 @@ public class AndroidFileBackendTest extends BackendTestBase { } @Test + public void managedUris_isSerializedAsIntegerOnDisk() throws Exception { + Account account = new Account("<internal>@gmail.com", "google.com"); + AccountManager mockManager = mock(AccountManager.class); + when(mockManager.getAccountId(account)).thenReturn(Futures.immediateFuture(123)); + + AndroidFileBackend backend = + AndroidFileBackend.builder(context).setAccountManager(mockManager).build(); + SynchronousFileStorage storage = new SynchronousFileStorage(ImmutableList.of(backend)); + + Uri uri = + Uri.parse( + "android://" + + context.getPackageName() + + "/managed/common/google.com%3Ayou%40gmail.com/file"); + createFile(storage, uri, TEST_CONTENT); + assertThat(storage.exists(uri)).isTrue(); + + File file = new File(context.getFilesDir(), "managed/common/123/file"); + assertThat(file.exists()).isTrue(); + } + + @Test + public void managedLocation_worksWithChildren() throws Exception { + Account account = new Account("<internal>@gmail.com", "google.com"); + AccountManager mockManager = mock(AccountManager.class); + when(mockManager.getAccount(123)).thenReturn(Futures.immediateFuture(account)); + when(mockManager.getAccountId(account)).thenReturn(Futures.immediateFuture(123)); + + AndroidFileBackend backend = + AndroidFileBackend.builder(context).setAccountManager(mockManager).build(); + + Uri dirUri = + Uri.parse( + "android://" + + context.getPackageName() + + "/managed/common/google.com%3Ayou%40gmail.com/dir"); + Uri fileUri0 = Uri.withAppendedPath(dirUri, "file-0"); + Uri fileUri1 = Uri.withAppendedPath(dirUri, "file-1"); + backend.createDirectory(dirUri); + backend.openForWrite(fileUri0).close(); + backend.openForWrite(fileUri1).close(); + + assertThat(backend.children(dirUri)).containsExactly(fileUri0, fileUri1); + } + + @Test + public void managedUris_worksWithToFile() throws Exception { + Account account = new Account("<internal>@gmail.com", "google.com"); + AccountManager mockManager = mock(AccountManager.class); + when(mockManager.getAccountId(account)).thenReturn(Futures.immediateFuture(123)); + + AndroidFileBackend backend = + AndroidFileBackend.builder(context).setAccountManager(mockManager).build(); + + Uri uri = + Uri.parse( + "android://" + + context.getPackageName() + + "/managed/common/google.com%3Ayou%40gmail.com/file"); + File file = backend.toFile(uri); + assertThat(file.getPath()).endsWith("/files/managed/common/123/file"); + } + + @Test public void lockScope_returnsNonNullLockScope() throws IOException { assertThat(backend.lockScope()).isNotNull(); } diff --git a/javatests/com/google/android/libraries/mobiledatadownload/file/backends/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/file/backends/BUILD index 1bf30c4..ffb042b 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/file/backends/BUILD +++ b/javatests/com/google/android/libraries/mobiledatadownload/file/backends/BUILD @@ -15,6 +15,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_binary", "android load("//java/com/google/android/libraries/mobiledatadownload/file/common/testing:build_defs.bzl", "android_test_multi_api") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -53,6 +54,7 @@ android_local_test( android_test_multi_api( name = "AssetFileBackendTest", size = "small", + timeout = "moderate", srcs = ["AssetFileBackendTest.java"], assets = [":test_assets"], assets_dir = "assets", @@ -107,7 +109,8 @@ android_local_test( "//java/com/google/android/libraries/mobiledatadownload/file/backends:account_manager", "//java/com/google/android/libraries/mobiledatadownload/file/backends:android", "//java/com/google/android/libraries/mobiledatadownload/file/backends:android_file_environment", - "@androidx_test", + "//java/com/google/android/libraries/mobiledatadownload/file/common/testing", + "//java/com/google/android/libraries/mobiledatadownload/file/common/testing:robolectric", "@com_google_guava_guava", "@mockito", "@truth", @@ -119,14 +122,17 @@ android_binary( testonly = 1, srcs = ["BlobStoreBackendTest.java"], manifest = "//javatests/com/google/android/libraries/mobiledatadownload/file:AndroidManifest.xml", + multidex = "legacy", deps = [ "//java/com/google/android/libraries/mobiledatadownload/file/backends:blob_uri", "//java/com/google/android/libraries/mobiledatadownload/file/backends:blobstore_backend", "//java/com/google/android/libraries/mobiledatadownload/file/backends:file_descriptor", "//java/com/google/android/libraries/mobiledatadownload/file/common", "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:charsets", + "//java/com/google/android/libraries/mobiledatadownload/file/common/testing", + "@android_sdk_linux", "@androidx_test", - "@com_google_android_testing//:testrunner", + "@com_google_android_testing//:testrunner", # unuseddeps: keep "@com_google_guava_guava", "@junit", "@truth", diff --git a/javatests/com/google/android/libraries/mobiledatadownload/file/behaviors/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/file/behaviors/BUILD index 5fbb304..3d49770 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/file/behaviors/BUILD +++ b/javatests/com/google/android/libraries/mobiledatadownload/file/behaviors/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_binary", "android_instrumentation_test", "android_library", "android_local_test") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -27,9 +28,11 @@ android_local_test( "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/file/backends:file", "//java/com/google/android/libraries/mobiledatadownload/file/behaviors:compute_uri", + "//java/com/google/android/libraries/mobiledatadownload/file/common/testing", "//java/com/google/android/libraries/mobiledatadownload/file/common/testing:extras", "//java/com/google/android/libraries/mobiledatadownload/file/openers:stream", "//java/com/google/android/libraries/mobiledatadownload/file/spi", + "//java/com/google/testing/mockito", "@com_google_guava_guava", "@mockito", "@truth", @@ -49,10 +52,12 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file/behaviors:syncing", "//java/com/google/android/libraries/mobiledatadownload/file/common/testing", "//java/com/google/android/libraries/mobiledatadownload/file/openers:stream", + "//java/com/google/android/libraries/mobiledatadownload/file/openers:string", "//java/com/google/android/libraries/mobiledatadownload/file/transforms:buffer", "@com_google_android_testing//:testrunner", "@com_google_guava_guava", "@junit", + "@mockito", "@truth", ], ) @@ -61,7 +66,11 @@ android_binary( name = "SyncingBehaviorAndroidTest_bin", testonly = 1, manifest = "//javatests/com/google/android/libraries/mobiledatadownload/file:AndroidManifest.xml", - deps = [":SyncingBehaviorAndroidTest_lib"], + multidex = "legacy", + deps = [ + ":SyncingBehaviorAndroidTest_lib", + "@android_sdk_linux", + ], ) android_instrumentation_test( diff --git a/javatests/com/google/android/libraries/mobiledatadownload/file/common/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/file/common/BUILD index 948a1ff..1f54388 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/file/common/BUILD +++ b/javatests/com/google/android/libraries/mobiledatadownload/file/common/BUILD @@ -11,9 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +load("//java/com/google/android/libraries/mobiledatadownload/file/common/testing:build_defs.bzl", "android_test_multi_api") load("@build_bazel_rules_android//android:rules.bzl", "android_local_test") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -38,12 +40,21 @@ android_local_test( ], ) -android_local_test( +android_test_multi_api( name = "LockScopeTest", size = "small", + timeout = "moderate", srcs = ["LockScopeTest.java"], + manifest = "LockScopeTestManifest.xml", deps = [ + "//java/com/google/android/libraries/mobiledatadownload/file", + "//java/com/google/android/libraries/mobiledatadownload/file/backends:file", + "//java/com/google/android/libraries/mobiledatadownload/file/backends:file_adapter", "//java/com/google/android/libraries/mobiledatadownload/file/common", + "//java/com/google/android/libraries/mobiledatadownload/file/common/testing", + "@androidx_test", + "@com_google_guava_guava", + "@junit", "@truth", ], ) diff --git a/javatests/com/google/android/libraries/mobiledatadownload/file/common/LockScopeTest.java b/javatests/com/google/android/libraries/mobiledatadownload/file/common/LockScopeTest.java index 67fe88d..8e3cd1f 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/file/common/LockScopeTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/file/common/LockScopeTest.java @@ -18,24 +18,40 @@ package com.google.android.libraries.mobiledatadownload.file.common; import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertThrows; +import android.content.Context; import android.net.Uri; -import com.google.thirdparty.robolectric.GoogleRobolectricTestRunner; +import androidx.test.core.app.ApplicationProvider; +import com.google.android.libraries.mobiledatadownload.file.backends.FileUriAdapter; +import com.google.android.libraries.mobiledatadownload.file.common.testing.TemporaryUri; +import com.google.common.io.Files; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Semaphore; +import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; -@RunWith(GoogleRobolectricTestRunner.class) +@RunWith(JUnit4.class) public class LockScopeTest { + // Keys to message data sent between main and service processes + private static final String URI_BUNDLE_KEY_1 = "uri1"; + private static final String URI_BUNDLE_KEY_2 = "uri2"; + + @Rule public final TemporaryUri tmpUri = new TemporaryUri(); + + private final Context mainContext = ApplicationProvider.getApplicationContext(); + @Test public void createWithSharedThreadLocks_sharesThreadLocksAcrossInstances() throws IOException { ConcurrentMap<String, Semaphore> lockMap = new ConcurrentHashMap<>(); LockScope lockScope = LockScope.createWithExistingThreadLocks(lockMap); LockScope otherLockScope = LockScope.createWithExistingThreadLocks(lockMap); - Uri uri = Uri.parse("file:///dummy"); + Uri uri = tmpUri.newUri(); try (Lock lock = lockScope.threadLock(uri)) { assertThat(otherLockScope.tryThreadLock(uri)).isNull(); @@ -47,9 +63,26 @@ public class LockScopeTest { @Test public void createWithFailingThreadLocks_willFailToAcquireThreadLocks() throws IOException { LockScope lockScope = LockScope.createWithFailingThreadLocks(); - Uri uri = Uri.parse("file:///dummy"); + Uri uri = tmpUri.newUri(); assertThrows(UnsupportedFileStorageOperation.class, () -> lockScope.threadLock(uri)); assertThat(lockScope.tryThreadLock(uri)).isNull(); } + + @Test + public void createFileLockSucceedsInSingleProcess() throws Exception { + LockScope lockScope = LockScope.create(); + Uri uri = tmpUri.newUri(); + + try (FileOutputStream stream = getStreamFromUri(uri); + Lock lock = lockScope.fileLock(stream.getChannel(), /* shared= */ false)) { + assertThat(lock).isNotNull(); + } + } + + private static FileOutputStream getStreamFromUri(Uri uri) throws IOException { + File file = FileUriAdapter.instance().toFile(uri); + Files.createParentDirs(file); + return new FileOutputStream(file); + } } diff --git a/javatests/com/google/android/libraries/mobiledatadownload/file/common/LockScopeTestManifest.xml b/javatests/com/google/android/libraries/mobiledatadownload/file/common/LockScopeTestManifest.xml new file mode 100644 index 0000000..243be84 --- /dev/null +++ b/javatests/com/google/android/libraries/mobiledatadownload/file/common/LockScopeTestManifest.xml @@ -0,0 +1,31 @@ +<!-- +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.google.android.libraries.mobiledatadownload.file.common"> + <uses-sdk android:minSdkVersion="15" android:targetSdkVersion="29"/> + + <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" /> + <application android:name="android.support.multidex.MultiDexApplication"> + <uses-library android:name="android.test.runner" /> + + </application> + <instrumentation + android:name="com.google.android.apps.common.testing.testrunner.Google3InstrumentationTestRunner" + android:targetPackage="com.google.android.libraries.mobiledatadownload.file.common" /> +</manifest>
\ No newline at end of file diff --git a/javatests/com/google/android/libraries/mobiledatadownload/file/common/internal/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/file/common/internal/BUILD index 396d6a6..c494d6d 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/file/common/internal/BUILD +++ b/javatests/com/google/android/libraries/mobiledatadownload/file/common/internal/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_local_test") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -41,6 +42,17 @@ android_local_test( ) android_local_test( + name = "ExponentialBackoffIteratorTest", + size = "small", + srcs = ["ExponentialBackoffIteratorTest.java"], + deps = [ + "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:exponential_backoff_iterator", + "@androidx_test", + "@truth", + ], +) + +android_local_test( name = "LazyByteArrayInputStreamTest", size = "small", srcs = ["LazyByteArrayInputStreamTest.java"], @@ -58,6 +70,7 @@ android_local_test( deps = [ "//java/com/google/android/libraries/mobiledatadownload/file/common:fragment", "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:lite_transform_fragments", + "//java/com/google/android/libraries/mobiledatadownload/file/common/testing", "@com_google_guava_guava", "@truth", ], diff --git a/javatests/com/google/android/libraries/mobiledatadownload/file/common/internal/ExponentialBackoffIteratorTest.java b/javatests/com/google/android/libraries/mobiledatadownload/file/common/internal/ExponentialBackoffIteratorTest.java new file mode 100644 index 0000000..150e043 --- /dev/null +++ b/javatests/com/google/android/libraries/mobiledatadownload/file/common/internal/ExponentialBackoffIteratorTest.java @@ -0,0 +1,79 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.file.common.internal; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import java.util.Iterator; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(AndroidJUnit4.class) +public final class ExponentialBackoffIteratorTest { + + @Test + public void testNegativeInitialBackoff() throws Exception { + assertThrows(IllegalArgumentException.class, () -> ExponentialBackoffIterator.create(-1, 0)); + } + + @Test + public void testZeroInitialBackoff() throws Exception { + assertThrows(IllegalArgumentException.class, () -> ExponentialBackoffIterator.create(0, 0)); + } + + @Test + public void testUpperBoundLessThanInitialBackoff() throws Exception { + assertThrows(IllegalArgumentException.class, () -> ExponentialBackoffIterator.create(1, 0)); + } + + @Test + public void testLargeInitialBackoffWillNotOverflow() throws Exception { + Iterator<Long> iterator = ExponentialBackoffIterator.create(Long.MAX_VALUE - 1, Long.MAX_VALUE); + + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(Long.MAX_VALUE - 1); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(Long.MAX_VALUE); + } + + @Test + public void testExponentialBackoffBackoffs() throws Exception { + Iterator<Long> iterator = ExponentialBackoffIterator.create(10, 1000); + + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(10); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(20); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(40); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(80); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(160); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(320); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(640); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(1000); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(1000); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(1000); + } +} diff --git a/javatests/com/google/android/libraries/mobiledatadownload/file/common/testing/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/file/common/testing/BUILD index 1861c30..5f7a15a 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/file/common/testing/BUILD +++ b/javatests/com/google/android/libraries/mobiledatadownload/file/common/testing/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_local_test") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -23,10 +24,13 @@ android_local_test( size = "small", srcs = ["FakeFileBackendTest.java"], deps = [ + "//java/com/google/android/libraries/mobiledatadownload/file", + "//java/com/google/android/libraries/mobiledatadownload/file/backends:file", "//java/com/google/android/libraries/mobiledatadownload/file/common", "//java/com/google/android/libraries/mobiledatadownload/file/common/testing", "//java/com/google/android/libraries/mobiledatadownload/file/common/testing:extras", "//java/com/google/android/libraries/mobiledatadownload/file/spi", "@com_google_guava_guava", + "@truth", ], ) diff --git a/javatests/com/google/android/libraries/mobiledatadownload/file/integration/downloader/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/file/integration/downloader/BUILD index f054997..dca79f2 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/file/integration/downloader/BUILD +++ b/javatests/com/google/android/libraries/mobiledatadownload/file/integration/downloader/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_local_test") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -29,7 +30,6 @@ android_local_test( "//java/com/google/android/libraries/mobiledatadownload/file/integration/downloader:downloader2", "//java/com/google/android/libraries/mobiledatadownload/file/integration/downloader:downloader2_sp", "//java/com/google/android/libraries/mobiledatadownload/file/openers:bytes", - "@androidx_test", "@com_google_guava_guava", "@downloader", "@truth", diff --git a/javatests/com/google/android/libraries/mobiledatadownload/file/integration/downloader/DownloadDestinationOpenerTest.java b/javatests/com/google/android/libraries/mobiledatadownload/file/integration/downloader/DownloadDestinationOpenerTest.java index 5688d55..8804991 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/file/integration/downloader/DownloadDestinationOpenerTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/file/integration/downloader/DownloadDestinationOpenerTest.java @@ -61,9 +61,9 @@ public final class DownloadDestinationOpenerTest { @Rule public TemporaryUri tmpUri = new TemporaryUri(); - /* Run the same test suite on multiple implementations of the same interface. */ + /* Run the same test suite on two implementations of the same interface. */ private enum Implementation { - SHARED_PREFERENCES + SHARED_PREFERENCES, } @Parameters(name = "implementation={0}") @@ -89,12 +89,12 @@ public final class DownloadDestinationOpenerTest { // Create destination. DownloadMetadataStore store = createMetadataStore(); - DownloadDestinationOpener opener = DownloadDestinationOpener.create(store); + DownloadDestinationOpener opener = DownloadDestinationOpener.create(store, EXECUTOR_SERVICE); DownloadDestination destination = storage.open(fileUri, opener); // Asset that destination has initial, empty values. - assertThat(destination.numExistingBytes()).isEqualTo(0); - assertThat(destination.readMetadata()).isEqualTo(emptyMetadata); + assertThat(destination.numExistingBytes().get(TIMEOUT, SECONDS)).isEqualTo(0); + assertThat(destination.readMetadata().get(TIMEOUT, SECONDS)).isEqualTo(emptyMetadata); } @Test @@ -110,10 +110,11 @@ public final class DownloadDestinationOpenerTest { // Create destination and write data/metadata. DownloadMetadataStore store = createMetadataStore(); - DownloadDestinationOpener opener = DownloadDestinationOpener.create(store); + DownloadDestinationOpener opener = DownloadDestinationOpener.create(store, EXECUTOR_SERVICE); DownloadDestination destination = storage.open(fileUri, opener); - try (WritableByteChannel writeChannel = destination.openByteChannel(0, metadataToWrite)) { + try (WritableByteChannel writeChannel = + destination.openByteChannel(0, metadataToWrite).get(TIMEOUT, SECONDS)) { writeChannel.write(buffer); } @@ -125,8 +126,8 @@ public final class DownloadDestinationOpenerTest { assertThat(readContent).isEqualTo(CONTENT); // Assert that destination now reflects the latest state. - assertThat(destination.numExistingBytes()).isEqualTo(CONTENT.length); - assertThat(destination.readMetadata()).isEqualTo(metadataToWrite); + assertThat(destination.numExistingBytes().get(TIMEOUT, SECONDS)).isEqualTo(CONTENT.length); + assertThat(destination.readMetadata().get(TIMEOUT, SECONDS)).isEqualTo(metadataToWrite); } @Test @@ -142,12 +143,12 @@ public final class DownloadDestinationOpenerTest { // Create destination. DownloadMetadataStore store = createMetadataStore(); - DownloadDestinationOpener opener = DownloadDestinationOpener.create(store); + DownloadDestinationOpener opener = DownloadDestinationOpener.create(store, EXECUTOR_SERVICE); DownloadDestination destination = storage.open(fileUri, opener); // Assert that destination now reflects the latest state. - assertThat(destination.numExistingBytes()).isEqualTo(CONTENT.length); - assertThat(destination.readMetadata()).isEqualTo(expectedMetadata); + assertThat(destination.numExistingBytes().get(TIMEOUT, SECONDS)).isEqualTo(CONTENT.length); + assertThat(destination.readMetadata().get(TIMEOUT, SECONDS)).isEqualTo(expectedMetadata); } @Test @@ -170,11 +171,13 @@ public final class DownloadDestinationOpenerTest { // Create destination and write data/metadata. DownloadMetadataStore store = createMetadataStore(); - DownloadDestinationOpener opener = DownloadDestinationOpener.create(store); + DownloadDestinationOpener opener = DownloadDestinationOpener.create(store, EXECUTOR_SERVICE); DownloadDestination destination = storage.open(fileUri, opener); try (WritableByteChannel writeChannel = - destination.openByteChannel(destination.numExistingBytes(), metadataToWrite)) { + destination + .openByteChannel(destination.numExistingBytes().get(TIMEOUT, SECONDS), metadataToWrite) + .get(TIMEOUT, SECONDS)) { writeChannel.write(buffer); } @@ -185,8 +188,9 @@ public final class DownloadDestinationOpenerTest { assertThat(readContent).isEqualTo(expectedContent); // Assert that destination now reflects the latest state. - assertThat(destination.numExistingBytes()).isEqualTo(expectedContent.length); - assertThat(destination.readMetadata()).isEqualTo(metadataToWrite); + assertThat(destination.numExistingBytes().get(TIMEOUT, SECONDS)) + .isEqualTo(expectedContent.length); + assertThat(destination.readMetadata().get(TIMEOUT, SECONDS)).isEqualTo(metadataToWrite); } @Test @@ -201,14 +205,14 @@ public final class DownloadDestinationOpenerTest { // Create destination and clear. DownloadMetadataStore store = createMetadataStore(); - DownloadDestinationOpener opener = DownloadDestinationOpener.create(store); + DownloadDestinationOpener opener = DownloadDestinationOpener.create(store, EXECUTOR_SERVICE); DownloadDestination destination = storage.open(fileUri, opener); - destination.clear(); + destination.clear().get(TIMEOUT, SECONDS); // Assert that destination now reflects the latest state. - assertThat(destination.numExistingBytes()).isEqualTo(0); - assertThat(destination.readMetadata()).isEqualTo(emptyMetadata); + assertThat(destination.numExistingBytes().get(TIMEOUT, SECONDS)).isEqualTo(0); + assertThat(destination.readMetadata().get(TIMEOUT, SECONDS)).isEqualTo(emptyMetadata); } private DownloadMetadataStore createMetadataStore() throws Exception { diff --git a/javatests/com/google/android/libraries/mobiledatadownload/file/monitors/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/file/monitors/BUILD index 23895dc..6b13621 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/file/monitors/BUILD +++ b/javatests/com/google/android/libraries/mobiledatadownload/file/monitors/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_local_test") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) diff --git a/javatests/com/google/android/libraries/mobiledatadownload/file/openers/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/file/openers/BUILD index a770dea..1c53a43 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/file/openers/BUILD +++ b/javatests/com/google/android/libraries/mobiledatadownload/file/openers/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_application_test", "android_local_test") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -89,10 +90,15 @@ android_local_test( deps = [ "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/file/backends:file", + "//java/com/google/android/libraries/mobiledatadownload/file/common", + "//java/com/google/android/libraries/mobiledatadownload/file/common:fragment", + "//java/com/google/android/libraries/mobiledatadownload/file/common/testing", "//java/com/google/android/libraries/mobiledatadownload/file/common/testing:matchers", "//java/com/google/android/libraries/mobiledatadownload/file/openers:native", "//java/com/google/android/libraries/mobiledatadownload/file/spi", + "//java/com/google/android/libraries/mobiledatadownload/file/transforms:compress", "//java/com/google/android/libraries/mobiledatadownload/file/transforms:proto", + "//java/com/google/testing/mockito", "@com_google_guava_guava", "@mockito", "@truth", @@ -307,11 +313,38 @@ android_local_test( "//java/com/google/android/libraries/mobiledatadownload/file/openers:recursive_delete", "//java/com/google/android/libraries/mobiledatadownload/file/openers:string", "//java/com/google/android/libraries/mobiledatadownload/file/spi", + "@androidx_test", "@mockito", "@truth", ], ) +android_application_test( + name = "RecursiveDeleteOpenerAndroidTest", + size = "large", + srcs = [ + "RecursiveDeleteOpenerAndroidTest.java", + ], + manifest = "RecursiveDeleteOpenerAndroidManifest.xml", + target_devices = [ + "//tools/android/emulated_devices/generic_phone:google_23_x86", + ], + deps = [ + "//java/com/google/android/libraries/mobiledatadownload/file", + "//java/com/google/android/libraries/mobiledatadownload/file/backends:android", + "//java/com/google/android/libraries/mobiledatadownload/file/backends:android_adapter", + "//java/com/google/android/libraries/mobiledatadownload/file/common/testing", + "//java/com/google/android/libraries/mobiledatadownload/file/openers:lock_file", + "//java/com/google/android/libraries/mobiledatadownload/file/openers:recursive_delete", + "//java/com/google/android/libraries/mobiledatadownload/file/openers:stream_mutation", + "//java/com/google/android/libraries/mobiledatadownload/file/openers:string", + "@androidx_test", + "@com_google_guava_guava", + "@junit", + "@truth", + ], +) + android_local_test( name = "RecursiveSizeOpenerTest", srcs = [ @@ -325,6 +358,7 @@ android_local_test( "//java/com/google/android/libraries/mobiledatadownload/file/openers:recursive_size", "//java/com/google/android/libraries/mobiledatadownload/file/transforms:compress", "//java/com/google/android/libraries/mobiledatadownload/file/transforms:proto", + "@androidx_test", "@truth", ], ) @@ -450,12 +484,17 @@ android_local_test( ], deps = [ "//java/com/google/android/libraries/mobiledatadownload/file", + "//java/com/google/android/libraries/mobiledatadownload/file/backends:file", "//java/com/google/android/libraries/mobiledatadownload/file/behaviors:syncing", "//java/com/google/android/libraries/mobiledatadownload/file/common/testing", "//java/com/google/android/libraries/mobiledatadownload/file/common/testing:extras", "//java/com/google/android/libraries/mobiledatadownload/file/common/testing:test_message_java_proto_lite", "//java/com/google/android/libraries/mobiledatadownload/file/openers:proto", + "//java/com/google/android/libraries/mobiledatadownload/file/openers:stream", + "//java/com/google/android/libraries/mobiledatadownload/file/openers:string", "//java/com/google/android/libraries/mobiledatadownload/file/transforms:compress", + "@androidx_test", + "@com_google_protobuf//:protobuf_lite", "@mockito", "@truth", ], diff --git a/javatests/com/google/android/libraries/mobiledatadownload/file/openers/RecursiveDeleteOpenerAndroidManifest.xml b/javatests/com/google/android/libraries/mobiledatadownload/file/openers/RecursiveDeleteOpenerAndroidManifest.xml new file mode 100644 index 0000000..72007f2 --- /dev/null +++ b/javatests/com/google/android/libraries/mobiledatadownload/file/openers/RecursiveDeleteOpenerAndroidManifest.xml @@ -0,0 +1,30 @@ +<!-- +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.google.android.libraries.mobiledatadownload.file.openers"> + <uses-sdk + android:minSdkVersion="21" + android:targetSdkVersion="23"/> + <application> + <uses-library android:name="android.test.runner" /> + </application> + <instrumentation + android:name="com.google.android.apps.common.testing.testrunner.Google3InstrumentationTestRunner" + android:targetPackage="com.google.android.libraries.mobiledatadownload.file.openers" /> +</manifest> diff --git a/javatests/com/google/android/libraries/mobiledatadownload/file/openers/RecursiveDeleteOpenerAndroidTest.java b/javatests/com/google/android/libraries/mobiledatadownload/file/openers/RecursiveDeleteOpenerAndroidTest.java new file mode 100644 index 0000000..69d09ae --- /dev/null +++ b/javatests/com/google/android/libraries/mobiledatadownload/file/openers/RecursiveDeleteOpenerAndroidTest.java @@ -0,0 +1,68 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.file.openers; + +import static com.google.common.truth.Truth.assertThat; + +import android.content.Context; +import android.net.Uri; +import android.system.Os; +import androidx.test.core.app.ApplicationProvider; +import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; +import com.google.android.libraries.mobiledatadownload.file.backends.AndroidFileBackend; +import com.google.android.libraries.mobiledatadownload.file.backends.AndroidUriAdapter; +import com.google.android.libraries.mobiledatadownload.file.common.testing.TemporaryAndroidUri; +import java.util.Arrays; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class RecursiveDeleteOpenerAndroidTest { + @Rule + public TemporaryAndroidUri tmpAndroidUri = + new TemporaryAndroidUri(ApplicationProvider.getApplicationContext()); + + private final SynchronousFileStorage storage = + new SynchronousFileStorage( + Arrays.asList( + AndroidFileBackend.builder(ApplicationProvider.getApplicationContext()).build())); + + @Test + public void open_notFollowingSymlink() throws Exception { + Context context = ApplicationProvider.getApplicationContext(); + SynchronousFileStorage storage = + new SynchronousFileStorage(Arrays.asList(AndroidFileBackend.builder(context).build())); + + Uri rootDir = tmpAndroidUri.newDirectoryUri(); + Uri dir = Uri.withAppendedPath(rootDir, "dir"); + Uri file0 = Uri.withAppendedPath(dir, "a"); + assertThat(storage.open(file0, WriteStringOpener.create("junk"))).isNull(); + Uri linkDir = Uri.withAppendedPath(rootDir, "link"); + AndroidUriAdapter adapter = AndroidUriAdapter.forContext(context); + Os.symlink(adapter.toFile(dir).getAbsolutePath(), adapter.toFile(linkDir).getAbsolutePath()); + Uri fileInLinkDir = Uri.withAppendedPath(linkDir, "a"); + + assertThat(storage.exists(fileInLinkDir)).isTrue(); + + assertThat(storage.open(linkDir, RecursiveDeleteOpener.create().withNoFollowLinks())).isNull(); + + assertThat(storage.exists(file0)).isTrue(); + assertThat(storage.exists(linkDir)).isFalse(); + assertThat(storage.exists(fileInLinkDir)).isFalse(); + } +} diff --git a/javatests/com/google/android/libraries/mobiledatadownload/file/samples/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/file/samples/BUILD index 9652356..ef675b8 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/file/samples/BUILD +++ b/javatests/com/google/android/libraries/mobiledatadownload/file/samples/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_local_test") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) diff --git a/javatests/com/google/android/libraries/mobiledatadownload/file/samples/SamplesTest.java b/javatests/com/google/android/libraries/mobiledatadownload/file/samples/SamplesTest.java index 808734e..52222dd 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/file/samples/SamplesTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/file/samples/SamplesTest.java @@ -112,7 +112,7 @@ public final class SamplesTest { String text = "SOME ALL CAPS TEXT"; createFile(storage, uri, text); try (InputStream in = storage.open(uri, ReadStreamOpener.create())) { - assertThat(in instanceof Sizable).isTrue(); + assertThat(in).isInstanceOf(Sizable.class); assertThat(((Sizable) in).size()).isEqualTo(text.length()); } } diff --git a/javatests/com/google/android/libraries/mobiledatadownload/file/transforms/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/file/transforms/BUILD index 7b66e71..0e143b1 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/file/transforms/BUILD +++ b/javatests/com/google/android/libraries/mobiledatadownload/file/transforms/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library", "android_local_test") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) diff --git a/javatests/com/google/android/libraries/mobiledatadownload/file/transforms/testdata/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/file/transforms/testdata/BUILD index b1326e9..53667db 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/file/transforms/testdata/BUILD +++ b/javatests/com/google/android/libraries/mobiledatadownload/file/transforms/testdata/BUILD @@ -14,13 +14,22 @@ load("//tools/build_rules/text_to_binary:def.bzl", "proto_data") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) exports_files([ + # NOTE: generated by GMM Offline's voice biasing model storage system + "aes_gcm_ciphertext", # encrypt only + "aes_gcm_key", + "aes_gcm_plaintext", + "zlib_aes_gcm_ciphertext", # compress then encrypt # NOTE: generated by CompressTransformTest#compressGoldenFile "golden.deflate", + # NOTE: test files for ZipTransformTest#decompressZip + "zip_test.zip", + "zip_test_directory/zip_test_subdirectory/zip_test_target.txt", ]) proto_data( diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/AndroidManifest.xml b/javatests/com/google/android/libraries/mobiledatadownload/internal/AndroidManifest.xml index 82140c5..b596d88 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/internal/AndroidManifest.xml +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/AndroidManifest.xml @@ -15,7 +15,6 @@ * limitations under the License. */ --> -<?xml version="1.0" encoding="utf-8"?> <manifest xmlns:android="http://schemas.android.com/apk/res/android" package="com.google.android.libraries.mobiledatadownload.internal"> <!-- Set minSdkVersion to 21 because android.os.symlink is only available after api level 21. --> diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/internal/BUILD index d72ab91..442c6a3 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/internal/BUILD +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/BUILD @@ -16,6 +16,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") load("//java/com/google/android/libraries/mobiledatadownload/file/common/testing:build_defs.bzl", "android_test_multi_api") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -27,9 +28,13 @@ mdd_local_test( deps = [ ":MddTestUtil", "//java/com/google/android/libraries/mobiledatadownload:DownloadException", + "//java/com/google/android/libraries/mobiledatadownload:ExperimentationConfig", "//java/com/google/android/libraries/mobiledatadownload:FileSource", + "//java/com/google/android/libraries/mobiledatadownload:Flags", "//java/com/google/android/libraries/mobiledatadownload:SilentFeedback", + "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/file/common/testing", + "//java/com/google/android/libraries/mobiledatadownload/file/common/testing:fake_file_backend", "//java/com/google/android/libraries/mobiledatadownload/internal:ExpirationHandler", "//java/com/google/android/libraries/mobiledatadownload/internal:FileGroupManager", "//java/com/google/android/libraries/mobiledatadownload/internal:FileGroupsMetadata", @@ -38,19 +43,23 @@ mdd_local_test( "//java/com/google/android/libraries/mobiledatadownload/internal:MobileDataDownloadManager", "//java/com/google/android/libraries/mobiledatadownload/internal:SharedFileManager", "//java/com/google/android/libraries/mobiledatadownload/internal:SharedFilesMetadata", + "//java/com/google/android/libraries/mobiledatadownload/internal/collect", "//java/com/google/android/libraries/mobiledatadownload/internal/experimentation:DownloadStageManager", "//java/com/google/android/libraries/mobiledatadownload/internal/experimentation:NoOpDownloadStageManager", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:FileGroupStatsLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LoggingStateStore", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:NetworkLogger", - "//java/com/google/android/libraries/mobiledatadownload/internal/logging:NoOpLoggingState", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:SharedPreferencesLoggingState", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:StorageLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//java/com/google/android/libraries/mobiledatadownload/internal/util:FileGroupUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/util:SharedPreferencesUtil", "//javatests/com/google/android/libraries/mobiledatadownload/testing:FakeTimeSource", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:MddTestDependencies", "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFlags", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "//proto:transform_java_proto_lite", "@androidx_test", "@com_google_guava_guava", @@ -66,12 +75,12 @@ mdd_local_test( test_class = "com.google.android.libraries.mobiledatadownload.internal.DataFileGroupValidatorTest", deps = [ ":MddTestUtil", + "//java/com/google/android/libraries/mobiledatadownload:Flags", "//java/com/google/android/libraries/mobiledatadownload/internal:DataFileGroupValidator", "//java/com/google/android/libraries/mobiledatadownload/internal:Migrations", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFlags", "//proto:transform_java_proto_lite", - "@androidx_test", "@com_google_guava_guava", "@truth", ], @@ -84,7 +93,6 @@ mdd_local_test( deps = [ "//java/com/google/android/libraries/mobiledatadownload:SilentFeedback", "//java/com/google/android/libraries/mobiledatadownload/internal:Migrations", - "@androidx_test", "@truth", ], ) @@ -98,7 +106,9 @@ mdd_local_test( "//java/com/google/android/libraries/mobiledatadownload:AccountSource", "//java/com/google/android/libraries/mobiledatadownload:AggregateException", "//java/com/google/android/libraries/mobiledatadownload:DownloadException", + "//java/com/google/android/libraries/mobiledatadownload:ExperimentationConfig", "//java/com/google/android/libraries/mobiledatadownload:FileSource", + "//java/com/google/android/libraries/mobiledatadownload:Flags", "//java/com/google/android/libraries/mobiledatadownload:SilentFeedback", "//java/com/google/android/libraries/mobiledatadownload/account:AccountUtil", "//java/com/google/android/libraries/mobiledatadownload/file", @@ -112,17 +122,23 @@ mdd_local_test( "//java/com/google/android/libraries/mobiledatadownload/internal:SharedFilesMetadata", "//java/com/google/android/libraries/mobiledatadownload/internal:SharedPreferencesFileGroupsMetadata", "//java/com/google/android/libraries/mobiledatadownload/internal:SharedPreferencesSharedFilesMetadata", + "//java/com/google/android/libraries/mobiledatadownload/internal/collect", "//java/com/google/android/libraries/mobiledatadownload/internal/downloader:DownloaderCallbackImpl", "//java/com/google/android/libraries/mobiledatadownload/internal/downloader:MddFileDownloader", "//java/com/google/android/libraries/mobiledatadownload/internal/experimentation:DownloadStageManager", "//java/com/google/android/libraries/mobiledatadownload/internal/experimentation:NoOpDownloadStageManager", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:DownloadStateLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging/testing:FakeEventLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//java/com/google/android/libraries/mobiledatadownload/internal/util:DirectoryUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/util:FileGroupUtil", "//java/com/google/android/libraries/mobiledatadownload/monitor:DownloadProgressMonitor", "//javatests/com/google/android/libraries/mobiledatadownload/testing:FakeTimeSource", "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFlags", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", "@com_google_protobuf//:any_proto", "@com_google_protobuf//:protobuf_lite", @@ -139,11 +155,14 @@ mdd_local_test( test_class = "com.google.android.libraries.mobiledatadownload.internal.FileGroupsMetadataTest", deps = [ ":MddTestUtil", + "//java/com/google/android/libraries/mobiledatadownload:Flags", "//java/com/google/android/libraries/mobiledatadownload:SilentFeedback", "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/file/backends:android", + "//java/com/google/android/libraries/mobiledatadownload/file/openers:stream", "//java/com/google/android/libraries/mobiledatadownload/internal:FileGroupsMetadata", "//java/com/google/android/libraries/mobiledatadownload/internal:SharedPreferencesFileGroupsMetadata", + "//java/com/google/android/libraries/mobiledatadownload/internal/collect", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//java/com/google/android/libraries/mobiledatadownload/internal/util:DirectoryUtil", @@ -154,7 +173,9 @@ mdd_local_test( "//javatests/com/google/android/libraries/mobiledatadownload/testing:FakeTimeSource", "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFlags", "//proto:download_config_java_proto_lite", + "//proto:log_enums_java_proto_lite", "@androidx_test", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", "@mockito", "@truth", @@ -167,6 +188,7 @@ mdd_local_test( test_class = "com.google.android.libraries.mobiledatadownload.internal.ExpirationHandlerTest", deps = [ ":MddTestUtil", + "//java/com/google/android/libraries/mobiledatadownload:Flags", "//java/com/google/android/libraries/mobiledatadownload:SilentFeedback", "//java/com/google/android/libraries/mobiledatadownload/delta:DeltaDecoder", "//java/com/google/android/libraries/mobiledatadownload/file", @@ -178,6 +200,7 @@ mdd_local_test( "//java/com/google/android/libraries/mobiledatadownload/internal:SharedFilesMetadata", "//java/com/google/android/libraries/mobiledatadownload/internal:SharedPreferencesFileGroupsMetadata", "//java/com/google/android/libraries/mobiledatadownload/internal:SharedPreferencesSharedFilesMetadata", + "//java/com/google/android/libraries/mobiledatadownload/internal/collect", "//java/com/google/android/libraries/mobiledatadownload/internal/downloader:MddFileDownloader", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", @@ -185,6 +208,7 @@ mdd_local_test( "//java/com/google/android/libraries/mobiledatadownload/monitor:DownloadProgressMonitor", "//javatests/com/google/android/libraries/mobiledatadownload/testing:FakeTimeSource", "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFlags", + "//proto:log_enums_java_proto_lite", "@androidx_test", "@com_google_guava_guava", "@mockito", @@ -200,11 +224,14 @@ mdd_local_test( ":MddTestUtil", "//java/com/google/android/libraries/mobiledatadownload:DownloadException", "//java/com/google/android/libraries/mobiledatadownload:FileSource", + "//java/com/google/android/libraries/mobiledatadownload:Flags", "//java/com/google/android/libraries/mobiledatadownload:SilentFeedback", "//java/com/google/android/libraries/mobiledatadownload/delta:DeltaDecoder", "//java/com/google/android/libraries/mobiledatadownload/file", + "//java/com/google/android/libraries/mobiledatadownload/file/backends", "//java/com/google/android/libraries/mobiledatadownload/file/backends:android", "//java/com/google/android/libraries/mobiledatadownload/file/backends:blob_uri", + "//java/com/google/android/libraries/mobiledatadownload/file/common/testing", "//java/com/google/android/libraries/mobiledatadownload/file/spi", "//java/com/google/android/libraries/mobiledatadownload/file/transforms:compress", "//java/com/google/android/libraries/mobiledatadownload/internal:FileGroupsMetadata", @@ -213,7 +240,9 @@ mdd_local_test( "//java/com/google/android/libraries/mobiledatadownload/internal:SharedFileManager", "//java/com/google/android/libraries/mobiledatadownload/internal:SharedFilesMetadata", "//java/com/google/android/libraries/mobiledatadownload/internal:SharedPreferencesSharedFilesMetadata", + "//java/com/google/android/libraries/mobiledatadownload/internal/downloader:DeltaFileDownloaderCallbackImpl", "//java/com/google/android/libraries/mobiledatadownload/internal/downloader:DownloaderCallbackImpl", + "//java/com/google/android/libraries/mobiledatadownload/internal/downloader:FileNameUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/downloader:MddFileDownloader", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", @@ -222,7 +251,10 @@ mdd_local_test( "//java/com/google/android/libraries/mobiledatadownload/internal/util:SharedPreferencesUtil", "//java/com/google/android/libraries/mobiledatadownload/monitor:DownloadProgressMonitor", "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFlags", - "@androidx_test", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", + "//proto:transform_java_proto_lite", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", "@com_google_protobuf//:protobuf_lite", "@mockito", @@ -236,9 +268,11 @@ mdd_local_test( test_class = "com.google.android.libraries.mobiledatadownload.internal.SharedFilesMetadataTest", deps = [ ":MddTestUtil", + "//java/com/google/android/libraries/mobiledatadownload:Flags", "//java/com/google/android/libraries/mobiledatadownload:SilentFeedback", "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/file/backends:android", + "//java/com/google/android/libraries/mobiledatadownload/file/openers:stream", "//java/com/google/android/libraries/mobiledatadownload/internal:Migrations", "//java/com/google/android/libraries/mobiledatadownload/internal:SharedFilesMetadata", "//java/com/google/android/libraries/mobiledatadownload/internal:SharedPreferencesSharedFilesMetadata", @@ -247,8 +281,9 @@ mdd_local_test( "//java/com/google/android/libraries/mobiledatadownload/internal/util:SharedFilesMetadataUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/util:SharedPreferencesUtil", "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFlags", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "//proto:transform_java_proto_lite", - "@androidx_test", "@com_google_guava_guava", "@mockito", "@truth", @@ -260,12 +295,14 @@ android_library( testonly = 1, srcs = ["MddTestUtil.java"], deps = [ + "//java/com/google/android/libraries/mobiledatadownload:Flags", "//java/com/google/android/libraries/mobiledatadownload/internal:SharedFilesMetadata", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//proto:download_config_java_proto_lite", "//proto:transform_java_proto_lite", "@androidx_test", "@com_google_android_testing//:util", + "@com_google_errorprone_error_prone_annotations", "@com_google_protobuf//:protobuf_lite", "@truth", "@ub_uiautomator", @@ -317,13 +354,20 @@ android_test_multi_api( "//java/com/google/android/libraries/mobiledatadownload/internal/downloader:MddFileDownloader", "//java/com/google/android/libraries/mobiledatadownload/internal/experimentation:NoOpDownloadStageManager", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LoggingStateStore", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:SharedPreferencesLoggingState", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//java/com/google/android/libraries/mobiledatadownload/internal/util:DirectoryUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/util:FileGroupUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/util:SymlinkUtil", "//java/com/google/android/libraries/mobiledatadownload/monitor:DownloadProgressMonitor", + "//java/com/google/android/libraries/mobiledatadownload/monitor:NetworkUsageMonitor", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:BlockingFileDownloader", "//javatests/com/google/android/libraries/mobiledatadownload/testing:FakeTimeSource", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:MddTestDependencies", "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFlags", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "//proto:transform_java_proto_lite", "@android_sdk_linux", "@androidx_test", diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/DataFileGroupValidatorTest.java b/javatests/com/google/android/libraries/mobiledatadownload/internal/DataFileGroupValidatorTest.java index 78ee83d..20f2cea 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/internal/DataFileGroupValidatorTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/DataFileGroupValidatorTest.java @@ -153,6 +153,7 @@ public class DataFileGroupValidatorTest { @Test public void testAddGroupForDownload_zip() { flags.enableZipFolder = Optional.of(true); + Migrations.setCurrentVersion(context, FileKeyVersion.USE_CHECKSUM_ONLY); Migrations.setMigratedToNewFileKey(context, true); DataFileGroupInternal.Builder fileGroupBuilder = @@ -175,6 +176,7 @@ public class DataFileGroupValidatorTest { @Test public void testAddGroupForDownload_zip_featureOff() { flags.enableZipFolder = Optional.of(false); + Migrations.setCurrentVersion(context, FileKeyVersion.USE_CHECKSUM_ONLY); Migrations.setMigratedToNewFileKey(context, true); DataFileGroupInternal.Builder fileGroupBuilder = @@ -197,6 +199,7 @@ public class DataFileGroupValidatorTest { @Test public void testAddGroupForDownload_zip_noDownloadFileChecksum() { flags.enableZipFolder = Optional.of(true); + Migrations.setCurrentVersion(context, FileKeyVersion.USE_CHECKSUM_ONLY); Migrations.setMigratedToNewFileKey(context, true); DataFileGroupInternal.Builder fileGroupBuilder = @@ -218,6 +221,7 @@ public class DataFileGroupValidatorTest { @Test public void testAddGroupForDownload_zip_targetOneFile() { flags.enableZipFolder = Optional.of(true); + Migrations.setCurrentVersion(context, FileKeyVersion.USE_CHECKSUM_ONLY); Migrations.setMigratedToNewFileKey(context, true); DataFileGroupInternal.Builder fileGroupBuilder = @@ -241,6 +245,7 @@ public class DataFileGroupValidatorTest { @Test public void testAddGroupForDownload_zip_moreThanOneTransforms() { flags.enableZipFolder = Optional.of(true); + Migrations.setCurrentVersion(context, FileKeyVersion.USE_CHECKSUM_ONLY); Migrations.setMigratedToNewFileKey(context, true); DataFileGroupInternal.Builder fileGroupBuilder = diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/ExpirationHandlerTest.java b/javatests/com/google/android/libraries/mobiledatadownload/internal/ExpirationHandlerTest.java index 863c39e..bdb19c1 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/internal/ExpirationHandlerTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/ExpirationHandlerTest.java @@ -26,13 +26,21 @@ import static org.mockito.Mockito.when; import android.content.Context; import android.net.Uri; -import android.util.Pair; import androidx.test.core.app.ApplicationProvider; +import com.google.mobiledatadownload.internal.MetadataProto.DataFile; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupBookkeeping; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal.AllowedReaders; +import com.google.mobiledatadownload.internal.MetadataProto.FileStatus; +import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; +import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey; +import com.google.mobiledatadownload.internal.MetadataProto.SharedFile; import com.google.android.libraries.mobiledatadownload.SilentFeedback; import com.google.android.libraries.mobiledatadownload.delta.DeltaDecoder; import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; import com.google.android.libraries.mobiledatadownload.file.spi.Backend; import com.google.android.libraries.mobiledatadownload.internal.Migrations.FileKeyVersion; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup; import com.google.android.libraries.mobiledatadownload.internal.downloader.MddFileDownloader; import com.google.android.libraries.mobiledatadownload.internal.logging.EventLogger; import com.google.android.libraries.mobiledatadownload.internal.util.DirectoryUtil; @@ -43,14 +51,7 @@ import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.MoreExecutors; -import com.google.mobiledatadownload.internal.MetadataProto.DataFile; -import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupBookkeeping; -import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; -import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal.AllowedReaders; -import com.google.mobiledatadownload.internal.MetadataProto.FileStatus; -import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; -import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey; -import com.google.mobiledatadownload.internal.MetadataProto.SharedFile; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -161,6 +162,7 @@ public final class ExpirationHandlerTest { "android://com.google.android.libraries.mobiledatadownload.internal/files/datadownload/shared/links/public/test-group-2/test-group-2_0"); private final TestFlags flags = new TestFlags(); + @Rule public final MockitoRule mocks = MockitoJUnit.rule(); @Before @@ -281,6 +283,7 @@ public final class ExpirationHandlerTest { verify(mockBackend).children(baseDownloadDirectoryUri); verify(mockBackend, never()).deleteFile(any()); verifyNoMoreInteractions(mockSharedFileManager); + verifyNoMoreInteractions(mockEventLogger); } @Test @@ -288,8 +291,8 @@ public final class ExpirationHandlerTest { DataFileGroupInternal dataFileGroup = MddTestUtil.createDataFileGroupInternal(TEST_GROUP_1, 2); NewFileKey[] fileKeys = MddTestUtil.createFileKeysForDataFileGroupInternal(dataFileGroup); - List<Pair<GroupKey, DataFileGroupInternal>> groups = - Arrays.asList(Pair.create(TEST_KEY_1, dataFileGroup)); + List<GroupKeyAndGroup> groups = + Arrays.asList(GroupKeyAndGroup.create(TEST_KEY_1, dataFileGroup)); when(mockFileGroupsMetadata.getAllFreshGroups()).thenReturn(Futures.immediateFuture(groups)); when(mockSharedFileManager.getFileStatus(fileKeys[0])) .thenReturn(Futures.immediateFuture(FileStatus.DOWNLOAD_COMPLETE)); @@ -321,6 +324,7 @@ public final class ExpirationHandlerTest { verify(mockBackend).isDirectory(dirFor1p); verify(mockBackend, never()).deleteFile(any()); verifyNoMoreInteractions(mockSharedFileManager); + verifyNoMoreInteractions(mockEventLogger); } @Test @@ -335,8 +339,8 @@ public final class ExpirationHandlerTest { NewFileKey[] fileKeys = MddTestUtil.createFileKeysForDataFileGroupInternal(dataFileGroup); - List<Pair<GroupKey, DataFileGroupInternal>> groups = - Arrays.asList(Pair.create(TEST_KEY_1, dataFileGroup)); + List<GroupKeyAndGroup> groups = + Arrays.asList(GroupKeyAndGroup.create(TEST_KEY_1, dataFileGroup)); when(mockFileGroupsMetadata.getAllFreshGroups()).thenReturn(Futures.immediateFuture(groups)); when(mockSharedFileManager.getFileStatus(fileKeys[0])) .thenReturn(Futures.immediateFuture(FileStatus.DOWNLOAD_COMPLETE)); @@ -368,6 +372,7 @@ public final class ExpirationHandlerTest { verify(mockBackend).isDirectory(dirFor1p); verify(mockBackend, never()).deleteFile(any()); verifyNoMoreInteractions(mockSharedFileManager); + verifyNoMoreInteractions(mockEventLogger); } @Test @@ -384,8 +389,8 @@ public final class ExpirationHandlerTest { NewFileKey[] fileKeys = MddTestUtil.createFileKeysForDataFileGroupInternal(dataFileGroup); - List<Pair<GroupKey, DataFileGroupInternal>> groups = - Arrays.asList(Pair.create(TEST_KEY_1, dataFileGroup)); + List<GroupKeyAndGroup> groups = + Arrays.asList(GroupKeyAndGroup.create(TEST_KEY_1, dataFileGroup)); when(mockFileGroupsMetadata.getAllFreshGroups()) .thenReturn(Futures.immediateFuture(groups)) .thenReturn(Futures.immediateFuture(new ArrayList<>())); @@ -437,6 +442,17 @@ public final class ExpirationHandlerTest { verify(mockBackend).deleteFile(testUri3); verify(mockBackend).deleteFile(testUri4); verifyNoMoreInteractions(mockSharedFileManager); + + verify(mockEventLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + dataFileGroup.getGroupName(), + dataFileGroup.getFileGroupVersionNumber(), + dataFileGroup.getBuildId(), + dataFileGroup.getVariantId()); + verify(mockEventLogger).logMddDataDownloadFileExpirationEvent(0, 4); + verify(mockEventLogger).logMddDataDownloadFileExpirationEvent(0, 5); + verifyNoMoreInteractions(mockEventLogger); } @Test @@ -451,8 +467,8 @@ public final class ExpirationHandlerTest { NewFileKey[] fileKeys = MddTestUtil.createFileKeysForDataFileGroupInternal(dataFileGroup); - List<Pair<GroupKey, DataFileGroupInternal>> groups = - Arrays.asList(Pair.create(TEST_KEY_1, dataFileGroup)); + List<GroupKeyAndGroup> groups = + Arrays.asList(GroupKeyAndGroup.create(TEST_KEY_1, dataFileGroup)); when(mockFileGroupsMetadata.getAllFreshGroups()).thenReturn(Futures.immediateFuture(groups)); when(mockSharedFileManager.getFileStatus(fileKeys[0])) .thenReturn(Futures.immediateFuture(FileStatus.DOWNLOAD_COMPLETE)); @@ -484,6 +500,7 @@ public final class ExpirationHandlerTest { verify(mockBackend).isDirectory(dirFor1p); verify(mockBackend, never()).deleteFile(any()); verifyNoMoreInteractions(mockSharedFileManager); + verifyNoMoreInteractions(mockEventLogger); } @Test @@ -499,8 +516,8 @@ public final class ExpirationHandlerTest { NewFileKey[] fileKeys = createFileKeysUseChecksumOnly(dataFileGroup); - List<Pair<GroupKey, DataFileGroupInternal>> groups = - Arrays.asList(Pair.create(TEST_KEY_1, dataFileGroup)); + List<GroupKeyAndGroup> groups = + Arrays.asList(GroupKeyAndGroup.create(TEST_KEY_1, dataFileGroup)); when(mockFileGroupsMetadata.getAllFreshGroups()).thenReturn(Futures.immediateFuture(groups)); when(mockSharedFileManager.getFileStatus(fileKeys[0])) .thenReturn(Futures.immediateFuture(FileStatus.DOWNLOAD_COMPLETE)); @@ -534,6 +551,7 @@ public final class ExpirationHandlerTest { verify(mockBackend).isDirectory(dirFor0p); verify(mockBackend, never()).deleteFile(any()); verifyNoMoreInteractions(mockSharedFileManager); + verifyNoMoreInteractions(mockEventLogger); } @Test @@ -552,8 +570,8 @@ public final class ExpirationHandlerTest { NewFileKey[] fileKeys = createFileKeysUseChecksumOnly(dataFileGroup); - List<Pair<GroupKey, DataFileGroupInternal>> groups = - Arrays.asList(Pair.create(TEST_KEY_1, dataFileGroup)); + List<GroupKeyAndGroup> groups = + Arrays.asList(GroupKeyAndGroup.create(TEST_KEY_1, dataFileGroup)); when(mockFileGroupsMetadata.getAllFreshGroups()) .thenReturn(Futures.immediateFuture(groups)) .thenReturn(Futures.immediateFuture(new ArrayList<>())); @@ -583,6 +601,17 @@ public final class ExpirationHandlerTest { verify(mockBackend).deleteFile(testUri2); verify(mockBackend).deleteFile(tempTestUri2); verifyNoMoreInteractions(mockSharedFileManager); + + verify(mockEventLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + dataFileGroup.getGroupName(), + dataFileGroup.getFileGroupVersionNumber(), + dataFileGroup.getBuildId(), + dataFileGroup.getVariantId()); + verify(mockEventLogger).logMddDataDownloadFileExpirationEvent(0, 2); + verify(mockEventLogger).logMddDataDownloadFileExpirationEvent(0, 1); + verifyNoMoreInteractions(mockEventLogger); } @Test @@ -614,8 +643,10 @@ public final class ExpirationHandlerTest { .setAllowedReadersEnum(AllowedReaders.ONLY_GOOGLE_PLAY_SERVICES) .build(); - List<Pair<GroupKey, DataFileGroupInternal>> groups = - Arrays.asList(Pair.create(TEST_KEY_1, firstGroup), Pair.create(TEST_KEY_2, secondGroup)); + List<GroupKeyAndGroup> groups = + Arrays.asList( + GroupKeyAndGroup.create(TEST_KEY_1, firstGroup), + GroupKeyAndGroup.create(TEST_KEY_2, secondGroup)); when(mockFileGroupsMetadata.getAllFreshGroups()).thenReturn(Futures.immediateFuture(groups)); when(mockSharedFileManager.getFileStatus(fileKey)) .thenReturn(Futures.immediateFuture(FileStatus.DOWNLOAD_COMPLETE)); @@ -640,6 +671,7 @@ public final class ExpirationHandlerTest { verify(mockBackend).isDirectory(dirForAll); verify(mockBackend, never()).deleteFile(any()); verifyNoMoreInteractions(mockSharedFileManager); + verifyNoMoreInteractions(mockEventLogger); } @Test @@ -673,8 +705,10 @@ public final class ExpirationHandlerTest { .setExpirationDateSecs(sooner.getTimeInMillis() / 1000) .build(); - List<Pair<GroupKey, DataFileGroupInternal>> groups = - Arrays.asList(Pair.create(TEST_KEY_1, firstGroup), Pair.create(TEST_KEY_2, secondGroup)); + List<GroupKeyAndGroup> groups = + Arrays.asList( + GroupKeyAndGroup.create(TEST_KEY_1, firstGroup), + GroupKeyAndGroup.create(TEST_KEY_2, secondGroup)); when(mockFileGroupsMetadata.getAllFreshGroups()).thenReturn(Futures.immediateFuture(groups)); when(mockSharedFileManager.getFileStatus(fileKey)) .thenReturn(Futures.immediateFuture(FileStatus.DOWNLOAD_COMPLETE)); @@ -699,6 +733,7 @@ public final class ExpirationHandlerTest { verify(mockBackend).isDirectory(dirForAll); verify(mockBackend, never()).deleteFile(any()); verifyNoMoreInteractions(mockSharedFileManager); + verifyNoMoreInteractions(mockEventLogger); } @Test @@ -747,6 +782,7 @@ public final class ExpirationHandlerTest { verify(mockBackend).isDirectory(dirFor1p); verify(mockBackend, never()).deleteFile(any()); verifyNoMoreInteractions(mockSharedFileManager); + verifyNoMoreInteractions(mockEventLogger); } @Test @@ -796,6 +832,7 @@ public final class ExpirationHandlerTest { verify(mockBackend).isDirectory(dirFor1p); verify(mockBackend, never()).deleteFile(any()); verifyNoMoreInteractions(mockSharedFileManager); + verifyNoMoreInteractions(mockEventLogger); } @Test @@ -847,6 +884,16 @@ public final class ExpirationHandlerTest { verify(mockBackend).isDirectory(testUri2); verify(mockBackend).deleteFile(testUri1); verify(mockBackend).deleteFile(testUri2); + verify(mockEventLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + dataFileGroup.getGroupName(), + dataFileGroup.getFileGroupVersionNumber(), + dataFileGroup.getBuildId(), + dataFileGroup.getVariantId()); + verify(mockEventLogger).logMddDataDownloadFileExpirationEvent(0, 2); + verify(mockEventLogger).logMddDataDownloadFileExpirationEvent(0, 2); + verifyNoMoreInteractions(mockEventLogger); } @Test @@ -900,6 +947,16 @@ public final class ExpirationHandlerTest { verify(mockBackend).isDirectory(testUri2); verify(mockBackend).deleteFile(testUri1); verify(mockBackend).deleteFile(testUri2); + verify(mockEventLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + dataFileGroup.getGroupName(), + dataFileGroup.getFileGroupVersionNumber(), + dataFileGroup.getBuildId(), + dataFileGroup.getVariantId()); + verify(mockEventLogger).logMddDataDownloadFileExpirationEvent(0, 2); + verify(mockEventLogger).logMddDataDownloadFileExpirationEvent(0, 2); + verifyNoMoreInteractions(mockEventLogger); } @Test @@ -955,6 +1012,16 @@ public final class ExpirationHandlerTest { verify(mockBackend).deleteFile(testDirFileUri1); verify(mockBackend).deleteFile(testDirFileUri2); verify(mockBackend).deleteFile(testUri2); + verify(mockEventLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + dataFileGroup.getGroupName(), + dataFileGroup.getFileGroupVersionNumber(), + dataFileGroup.getBuildId(), + dataFileGroup.getVariantId()); + verify(mockEventLogger).logMddDataDownloadFileExpirationEvent(0, 3); + verify(mockEventLogger).logMddDataDownloadFileExpirationEvent(0, 2); + verifyNoMoreInteractions(mockEventLogger); } @Test @@ -993,8 +1060,7 @@ public final class ExpirationHandlerTest { .setStaleLifetimeSecs((sooner.getTimeInMillis() - now.getTimeInMillis()) / 1000) .build(); - List<Pair<GroupKey, DataFileGroupInternal>> groups = - Arrays.asList(Pair.create(TEST_KEY_1, activeGroup)); + List<GroupKeyAndGroup> groups = Arrays.asList(GroupKeyAndGroup.create(TEST_KEY_1, activeGroup)); when(mockFileGroupsMetadata.getAllFreshGroups()).thenReturn(Futures.immediateFuture(groups)); fileGroupsMetadataStaleGroups.set(ImmutableList.of(staleGroup)); @@ -1019,6 +1085,7 @@ public final class ExpirationHandlerTest { verify(mockSharedFilesMetadata).getAllFileKeys(); verify(mockSharedFileManager).getOnDeviceUri(fileKey); verifyNoMoreInteractions(mockSharedFileManager); + verifyNoMoreInteractions(mockEventLogger); verify(mockBackend).exists(baseDownloadDirectoryUri); verify(mockBackend).children(baseDownloadDirectoryUri); verify(mockBackend).isDirectory(dirForAll); @@ -1059,8 +1126,7 @@ public final class ExpirationHandlerTest { .setStaleLifetimeSecs(laterTimeSecs - now.getTimeInMillis() / 1000) .build(); - List<Pair<GroupKey, DataFileGroupInternal>> groups = - Arrays.asList(Pair.create(TEST_KEY_1, activeGroup)); + List<GroupKeyAndGroup> groups = Arrays.asList(GroupKeyAndGroup.create(TEST_KEY_1, activeGroup)); when(mockFileGroupsMetadata.getAllFreshGroups()).thenReturn(Futures.immediateFuture(groups)); fileGroupsMetadataStaleGroups.set(ImmutableList.of(staleGroup)); @@ -1084,6 +1150,7 @@ public final class ExpirationHandlerTest { verify(mockSharedFilesMetadata).getAllFileKeys(); verify(mockSharedFileManager).getOnDeviceUri(fileKey); verifyNoMoreInteractions(mockSharedFileManager); + verifyNoMoreInteractions(mockEventLogger); verify(mockBackend).exists(baseDownloadDirectoryUri); verify(mockBackend).children(baseDownloadDirectoryUri); verify(mockBackend).isDirectory(dirForAll); @@ -1122,8 +1189,7 @@ public final class ExpirationHandlerTest { .setStaleLifetimeSecs((sooner.getTimeInMillis() - now.getTimeInMillis()) / 1000) .build(); - List<Pair<GroupKey, DataFileGroupInternal>> groups = - Arrays.asList(Pair.create(TEST_KEY_1, activeGroup)); + List<GroupKeyAndGroup> groups = Arrays.asList(GroupKeyAndGroup.create(TEST_KEY_1, activeGroup)); when(mockFileGroupsMetadata.getAllFreshGroups()).thenReturn(Futures.immediateFuture(groups)); fileGroupsMetadataStaleGroups.set(ImmutableList.of(staleGroup)); @@ -1147,6 +1213,7 @@ public final class ExpirationHandlerTest { verify(mockSharedFilesMetadata).getAllFileKeys(); verify(mockSharedFileManager).getOnDeviceUri(fileKey); verifyNoMoreInteractions(mockSharedFileManager); + verifyNoMoreInteractions(mockEventLogger); verify(mockBackend).exists(baseDownloadDirectoryUri); verify(mockBackend).children(baseDownloadDirectoryUri); verify(mockBackend).isDirectory(dirForAll); @@ -1187,8 +1254,7 @@ public final class ExpirationHandlerTest { .setStaleLifetimeSecs(laterTimeSecs - now.getTimeInMillis() / 1000) .build(); - List<Pair<GroupKey, DataFileGroupInternal>> groups = - Arrays.asList(Pair.create(TEST_KEY_1, activeGroup)); + List<GroupKeyAndGroup> groups = Arrays.asList(GroupKeyAndGroup.create(TEST_KEY_1, activeGroup)); when(mockFileGroupsMetadata.getAllFreshGroups()).thenReturn(Futures.immediateFuture(groups)); fileGroupsMetadataStaleGroups.set(ImmutableList.of(staleGroup)); @@ -1212,6 +1278,14 @@ public final class ExpirationHandlerTest { verify(mockSharedFilesMetadata).getAllFileKeys(); verify(mockSharedFileManager).getOnDeviceUri(fileKey); verifyNoMoreInteractions(mockSharedFileManager); + verify(mockEventLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + activeGroup.getGroupName(), + activeGroup.getFileGroupVersionNumber(), + activeGroup.getBuildId(), + activeGroup.getVariantId()); + verifyNoMoreInteractions(mockEventLogger); verify(mockBackend).exists(baseDownloadDirectoryUri); verify(mockBackend).children(baseDownloadDirectoryUri); verify(mockBackend).isDirectory(dirForAll); @@ -1249,8 +1323,7 @@ public final class ExpirationHandlerTest { .setExpirationDateSecs(earliestSecs) .build(); - List<Pair<GroupKey, DataFileGroupInternal>> groups = - Arrays.asList(Pair.create(TEST_KEY_1, firstGroup)); + List<GroupKeyAndGroup> groups = Arrays.asList(GroupKeyAndGroup.create(TEST_KEY_1, firstGroup)); when(mockFileGroupsMetadata.getAllFreshGroups()) .thenReturn(Futures.immediateFuture(groups)) .thenReturn(Futures.immediateFuture(ImmutableList.of())); @@ -1307,8 +1380,8 @@ public final class ExpirationHandlerTest { .setAllowedReadersEnum(AllowedReaders.ONLY_GOOGLE_PLAY_SERVICES) .setExpirationDateSecs(firstExpirationSecs); - List<Pair<GroupKey, DataFileGroupInternal>> groups = - Arrays.asList(Pair.create(TEST_KEY_1, firstGroup.build())); + List<GroupKeyAndGroup> groups = + Arrays.asList(GroupKeyAndGroup.create(TEST_KEY_1, firstGroup.build())); when(mockFileGroupsMetadata.getAllFreshGroups()).thenReturn(Futures.immediateFuture(groups)); when(mockSharedFileManager.getFileStatus(fileKey)) @@ -1384,7 +1457,7 @@ public final class ExpirationHandlerTest { .setExpirationDateSecs(secondExpirationSecs) .build(); - groups = Arrays.asList(Pair.create(TEST_KEY_2, secondGroup)); + groups = Arrays.asList(GroupKeyAndGroup.create(TEST_KEY_2, secondGroup)); when(mockFileGroupsMetadata.getAllFreshGroups()).thenReturn(Futures.immediateFuture(groups)); expirationHandler.updateExpiration().get(); @@ -1418,6 +1491,7 @@ public final class ExpirationHandlerTest { verify(mockSharedFileManager, times(4)).getOnDeviceUri(fileKey); verify(mockSharedFilesMetadata, times(4)).getAllFileKeys(); verifyNoMoreInteractions(mockSharedFileManager); + verifyNoMoreInteractions(mockEventLogger); verify(mockBackend, times(4)).exists(baseDownloadDirectoryUri); verify(mockBackend, times(4)).children(baseDownloadDirectoryUri); verify(mockBackend, times(4)).isDirectory(dirForAll); @@ -1454,6 +1528,17 @@ public final class ExpirationHandlerTest { verify(mockBlobStoreBackend).deleteFile(blobUri); assertThat(sharedFilesMetadata.read(fileKeys[0]).get()).isNull(); assertThat(fileGroupsMetadata.read(TEST_KEY_1).get()).isNull(); + verify(mockEventLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + dataFileGroup.getGroupName(), + dataFileGroup.getFileGroupVersionNumber(), + dataFileGroup.getBuildId(), + dataFileGroup.getVariantId()); + verify(mockEventLogger).logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); + verify(mockEventLogger).logMddDataDownloadFileExpirationEvent(0, 1); + verify(mockEventLogger).logMddDataDownloadFileExpirationEvent(0, 1); + verifyNoMoreInteractions(mockEventLogger); } @Test @@ -1489,6 +1574,16 @@ public final class ExpirationHandlerTest { verify(mockBlobStoreBackend).deleteFile(blobUri); assertThat(sharedFilesMetadata.read(fileKeys[0]).get()).isNull(); assertThat(fileGroupsMetadata.read(TEST_KEY_1).get()).isNull(); + verify(mockEventLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + dataFileGroup.getGroupName(), + dataFileGroup.getFileGroupVersionNumber(), + dataFileGroup.getBuildId(), + dataFileGroup.getVariantId()); + verify(mockEventLogger).logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); + verify(mockEventLogger).logMddDataDownloadFileExpirationEvent(0, 1); + verifyNoMoreInteractions(mockEventLogger); } @Test @@ -1524,6 +1619,19 @@ public final class ExpirationHandlerTest { assertThat(sharedFilesMetadata.read(fileKeys[0]).get()).isNull(); assertThat(sharedFilesMetadata.read(fileKeys[1]).get()).isNull(); assertThat(fileGroupsMetadata.read(TEST_KEY_1).get()).isNull(); + + verify(mockEventLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + dataFileGroup.getGroupName(), + dataFileGroup.getFileGroupVersionNumber(), + dataFileGroup.getBuildId(), + dataFileGroup.getVariantId()); + verify(mockEventLogger).logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); + verify(mockEventLogger).logMddDataDownloadFileExpirationEvent(0, 1); + verify(mockEventLogger).logMddDataDownloadFileExpirationEvent(0, 1); + verify(mockEventLogger).logMddDataDownloadFileExpirationEvent(0, 2); + verifyNoMoreInteractions(mockEventLogger); } @Test @@ -1555,6 +1663,8 @@ public final class ExpirationHandlerTest { assertThat(fileGroupsMetadata.read(TEST_KEY_1).get()).isNotNull(); verify(mockBlobStoreBackend, never()).deleteFile(blobUri); verify(mockBackend).deleteFile(tempTestUri2); + verify(mockEventLogger).logMddDataDownloadFileExpirationEvent(0, 1); + verifyNoMoreInteractions(mockEventLogger); } @Test @@ -1577,8 +1687,8 @@ public final class ExpirationHandlerTest { NewFileKey[] fileKeys = MddTestUtil.createFileKeysForDataFileGroupInternal(dataFileGroup); - List<Pair<GroupKey, DataFileGroupInternal>> groups = - Arrays.asList(Pair.create(TEST_KEY_1, dataFileGroup)); + List<GroupKeyAndGroup> groups = + Arrays.asList(GroupKeyAndGroup.create(TEST_KEY_1, dataFileGroup)); when(mockFileGroupsMetadata.getAllFreshGroups()) .thenReturn(Futures.immediateFuture(groups)) .thenReturn(Futures.immediateFuture(new ArrayList<>())); @@ -1621,8 +1731,8 @@ public final class ExpirationHandlerTest { NewFileKey[] fileKeys = MddTestUtil.createFileKeysForDataFileGroupInternal(dataFileGroup); // Setup mocks to return our fresh group - List<Pair<GroupKey, DataFileGroupInternal>> groups = - Arrays.asList(Pair.create(TEST_KEY_1, dataFileGroup)); + List<GroupKeyAndGroup> groups = + Arrays.asList(GroupKeyAndGroup.create(TEST_KEY_1, dataFileGroup)); when(mockFileGroupsMetadata.getAllFreshGroups()).thenReturn(Futures.immediateFuture(groups)); when(mockSharedFileManager.getFileStatus(fileKeys[0])) .thenReturn(Futures.immediateFuture(FileStatus.DOWNLOAD_COMPLETE)); @@ -1710,8 +1820,8 @@ public final class ExpirationHandlerTest { .build(); NewFileKey[] fileKeys = MddTestUtil.createFileKeysForDataFileGroupInternal(isolatedGroup1); - List<Pair<GroupKey, DataFileGroupInternal>> groups = - Arrays.asList(Pair.create(TEST_KEY_1, isolatedGroup1)); + List<GroupKeyAndGroup> groups = + Arrays.asList(GroupKeyAndGroup.create(TEST_KEY_1, isolatedGroup1)); when(mockFileGroupsMetadata.getAllFreshGroups()).thenReturn(Futures.immediateFuture(groups)); when(mockSharedFileManager.getFileStatus(fileKeys[0])) .thenReturn(Futures.immediateFuture(FileStatus.DOWNLOAD_COMPLETE)); diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/FileGroupManagerTest.java b/javatests/com/google/android/libraries/mobiledatadownload/internal/FileGroupManagerTest.java index 067bc81..7d7ad99 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/internal/FileGroupManagerTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/FileGroupManagerTest.java @@ -26,8 +26,10 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isA; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; @@ -37,8 +39,20 @@ import android.content.pm.PackageInfo; import android.content.pm.PackageManager; import android.net.Uri; import android.os.Build; -import android.util.Pair; import androidx.test.core.app.ApplicationProvider; +import com.google.mobiledatadownload.internal.MetadataProto.DataFile; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupBookkeeping; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal.AllowedReaders; +import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions; +import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions.ActivatingCondition; +import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions.DeviceNetworkPolicy; +import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions.DeviceStoragePolicy; +import com.google.mobiledatadownload.internal.MetadataProto.ExtraHttpHeader; +import com.google.mobiledatadownload.internal.MetadataProto.FileStatus; +import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; +import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey; +import com.google.mobiledatadownload.internal.MetadataProto.SharedFile; import com.google.android.libraries.mobiledatadownload.AccountSource; import com.google.android.libraries.mobiledatadownload.AggregateException; import com.google.android.libraries.mobiledatadownload.DownloadException; @@ -51,17 +65,21 @@ import com.google.android.libraries.mobiledatadownload.file.backends.AndroidFile import com.google.android.libraries.mobiledatadownload.file.common.LimitExceededException; import com.google.android.libraries.mobiledatadownload.file.spi.Backend; import com.google.android.libraries.mobiledatadownload.internal.FileGroupManager.GroupDownloadStatus; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup; import com.google.android.libraries.mobiledatadownload.internal.downloader.DownloaderCallbackImpl; import com.google.android.libraries.mobiledatadownload.internal.downloader.MddFileDownloader; import com.google.android.libraries.mobiledatadownload.internal.experimentation.DownloadStageManager; import com.google.android.libraries.mobiledatadownload.internal.experimentation.NoOpDownloadStageManager; +import com.google.android.libraries.mobiledatadownload.internal.logging.DownloadStateLogger; import com.google.android.libraries.mobiledatadownload.internal.logging.EventLogger; +import com.google.android.libraries.mobiledatadownload.internal.logging.testing.FakeEventLogger; import com.google.android.libraries.mobiledatadownload.internal.util.DirectoryUtil; import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupUtil; import com.google.android.libraries.mobiledatadownload.monitor.DownloadProgressMonitor; import com.google.android.libraries.mobiledatadownload.testing.FakeTimeSource; import com.google.android.libraries.mobiledatadownload.testing.TestFlags; import com.google.common.base.Optional; +import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -71,19 +89,9 @@ import com.google.common.util.concurrent.AsyncFunction; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; -import com.google.mobiledatadownload.internal.MetadataProto.DataFile; -import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupBookkeeping; -import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; -import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal.AllowedReaders; -import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions; -import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions.ActivatingCondition; -import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions.DeviceNetworkPolicy; -import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions.DeviceStoragePolicy; -import com.google.mobiledatadownload.internal.MetadataProto.ExtraHttpHeader; -import com.google.mobiledatadownload.internal.MetadataProto.FileStatus; -import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; -import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey; -import com.google.mobiledatadownload.internal.MetadataProto.SharedFile; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; +import com.google.mobiledatadownload.LogEnumsProto.MddDownloadResult; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; import com.google.protobuf.Any; import com.google.protobuf.ByteString; import com.google.protobuf.ExtensionRegistryLite; @@ -136,13 +144,14 @@ public class FileGroupManagerTest { private static final Correspondence<GroupKey, String> GROUP_KEY_TO_VARIANT = Correspondence.transforming(GroupKey::getVariantId, "using variant"); - private static final Correspondence<Pair<GroupKey, DataFileGroupInternal>, Pair<String, String>> - KEY_GROUP_PAIR_TO_VARIANT_PAIR = - Correspondence.transforming( - keyGroupPair -> - Pair.create( - keyGroupPair.first.getVariantId(), keyGroupPair.second.getVariantId()), - "using variants from group key and file group"); + private static final Correspondence<GroupKeyAndGroup, String> KEY_GROUP_PAIR_TO_VARIANT = + Correspondence.transforming( + keyGroupPair -> { + assertThat(keyGroupPair.groupKey().getVariantId()) + .isEqualTo(keyGroupPair.dataFileGroup().getVariantId()); + return keyGroupPair.dataFileGroup().getVariantId(); + }, + "using variant from group key and file group"); private static GroupKey testKey; private static GroupKey testKey2; @@ -158,8 +167,12 @@ public class FileGroupManagerTest { private SynchronousFileStorage fileStorage; public File publicDirectory; private final TestFlags flags = new TestFlags(); - @Rule public TemporaryFolder folder = new TemporaryFolder(); - @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + + @Rule(order = 2) + public TemporaryFolder folder = new TemporaryFolder(); + + @Rule(order = 3) + public final MockitoRule mocks = MockitoJUnit.rule(); @Mock EventLogger mockLogger; @Mock SilentFeedback mockSilentFeedback; @@ -260,8 +273,22 @@ public class FileGroupManagerTest { ReflectionHelpers.setStaticField(Build.VERSION.class, "SDK_INT", Build.VERSION_CODES.R); } + private void assertLoggedNewConfigs( + FakeEventLogger fakeEventLogger, + DataDownloadFileGroupStats fileGroupStats, + Void newConfigReceivedInfo) { + ArrayListMultimap<DataDownloadFileGroupStats, Void> loggedConfigs = + fakeEventLogger.getLoggedNewConfigReceived(); + assertThat(loggedConfigs).hasSize(1); + assertThat(loggedConfigs.get(fileGroupStats)).containsExactly(newConfigReceivedInfo); + } + @Test public void testAddGroupForDownload() throws Exception { + FakeEventLogger fakeEventLogger = new FakeEventLogger(); + + resetFileGroupManager(fakeEventLogger, fileGroupsMetadata, sharedFileManager); + DataFileGroupInternal dataFileGroup = MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 2); NewFileKey[] groupKeys = MddTestUtil.createFileKeysForDataFileGroupInternal(dataFileGroup); @@ -273,10 +300,16 @@ public class FileGroupManagerTest { assertThat(sharedFileManager.getSharedFile(groupKeys[0]).get()).isNotNull(); assertThat(sharedFileManager.getSharedFile(groupKeys[1]).get()).isNotNull(); + + assertLoggedNewConfigs( + fakeEventLogger, createFileGroupDetails(dataFileGroup).clearFileCount().build(), null); } @Test public void testAddGroupForDownload_correctlyPopulatesBuildIdAndVariantId() throws Exception { + FakeEventLogger fakeEventLogger = new FakeEventLogger(); + resetFileGroupManager(fakeEventLogger, fileGroupsMetadata, sharedFileManager); + DataFileGroupInternal dataFileGroup = MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 2).toBuilder() .setBuildId(10) @@ -292,14 +325,24 @@ public class FileGroupManagerTest { assertThat(sharedFileManager.getSharedFile(groupKeys[0]).get()).isNotNull(); assertThat(sharedFileManager.getSharedFile(groupKeys[1]).get()).isNotNull(); + + assertLoggedNewConfigs( + fakeEventLogger, createFileGroupDetails(dataFileGroup).clearFileCount().build(), null); } @Test public void testAddGroupForDownload_groupUpdated() throws Exception { + FakeEventLogger fakeEventLogger = new FakeEventLogger(); + resetFileGroupManager(fakeEventLogger, fileGroupsMetadata, sharedFileManager); + DataFileGroupInternal dataFileGroup = MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 2); assertThat(fileGroupManager.addGroupForDownload(testKey, dataFileGroup).get()).isTrue(); verifyAddGroupForDownloadWritesMetadata(testKey, dataFileGroup, CURRENT_TIMESTAMP); + assertLoggedNewConfigs( + fakeEventLogger, createFileGroupDetails(dataFileGroup).clearFileCount().build(), null); + fakeEventLogger.reset(); + // Update the file id and see that the group gets updated in the pending groups list. dataFileGroup = dataFileGroup.toBuilder() @@ -309,15 +352,27 @@ public class FileGroupManagerTest { assertThat(fileGroupManager.addGroupForDownload(testKey, dataFileGroup).get()).isTrue(); verifyAddGroupForDownloadWritesMetadata(testKey, dataFileGroup, CURRENT_TIMESTAMP); + assertLoggedNewConfigs( + fakeEventLogger, createFileGroupDetails(dataFileGroup).clearFileCount().build(), null); + fakeEventLogger.reset(); + // Update other parameters and check that we successfully add the group. dataFileGroup = dataFileGroup.toBuilder().setFileGroupVersionNumber(2).build(); assertThat(fileGroupManager.addGroupForDownload(testKey, dataFileGroup).get()).isTrue(); verifyAddGroupForDownloadWritesMetadata(testKey, dataFileGroup, CURRENT_TIMESTAMP); + assertLoggedNewConfigs( + fakeEventLogger, createFileGroupDetails(dataFileGroup).clearFileCount().build(), null); + fakeEventLogger.reset(); + dataFileGroup = dataFileGroup.toBuilder().setStaleLifetimeSecs(50).build(); assertThat(fileGroupManager.addGroupForDownload(testKey, dataFileGroup).get()).isTrue(); verifyAddGroupForDownloadWritesMetadata(testKey, dataFileGroup, CURRENT_TIMESTAMP); + assertLoggedNewConfigs( + fakeEventLogger, createFileGroupDetails(dataFileGroup).clearFileCount().build(), null); + fakeEventLogger.reset(); + dataFileGroup = dataFileGroup.toBuilder() .setDownloadConditions( @@ -328,6 +383,10 @@ public class FileGroupManagerTest { assertThat(fileGroupManager.addGroupForDownload(testKey, dataFileGroup).get()).isTrue(); verifyAddGroupForDownloadWritesMetadata(testKey, dataFileGroup, CURRENT_TIMESTAMP); + assertLoggedNewConfigs( + fakeEventLogger, createFileGroupDetails(dataFileGroup).clearFileCount().build(), null); + fakeEventLogger.reset(); + DownloadConditions downloadConditions = DownloadConditions.newBuilder() .setDeviceStoragePolicy(DeviceStoragePolicy.BLOCK_DOWNLOAD_LOWER_THRESHOLD) @@ -336,36 +395,61 @@ public class FileGroupManagerTest { assertThat(fileGroupManager.addGroupForDownload(testKey, dataFileGroup).get()).isTrue(); verifyAddGroupForDownloadWritesMetadata(testKey, dataFileGroup, CURRENT_TIMESTAMP); + assertLoggedNewConfigs( + fakeEventLogger, createFileGroupDetails(dataFileGroup).clearFileCount().build(), null); + fakeEventLogger.reset(); + dataFileGroup = dataFileGroup.toBuilder() .setAllowedReadersEnum(AllowedReaders.ONLY_GOOGLE_PLAY_SERVICES) .build(); assertThat(fileGroupManager.addGroupForDownload(testKey, dataFileGroup).get()).isTrue(); verifyAddGroupForDownloadWritesMetadata(testKey, dataFileGroup, CURRENT_TIMESTAMP); + + assertLoggedNewConfigs( + fakeEventLogger, createFileGroupDetails(dataFileGroup).clearFileCount().build(), null); } @Test public void testAddGroupForDownload_groupUpdated_whenBuildChanges() throws Exception { + FakeEventLogger fakeEventLogger = new FakeEventLogger(); + resetFileGroupManager(fakeEventLogger, fileGroupsMetadata, sharedFileManager); + DataFileGroupInternal dataFileGroup = MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 2); assertThat(fileGroupManager.addGroupForDownload(testKey, dataFileGroup).get()).isTrue(); verifyAddGroupForDownloadWritesMetadata(testKey, dataFileGroup, CURRENT_TIMESTAMP); + // Reset to clear events before next add group call + fakeEventLogger.reset(); + // Update the file id and see that the group gets updated in the pending groups list. dataFileGroup = dataFileGroup.toBuilder().setBuildId(123456789L).build(); assertThat(fileGroupManager.addGroupForDownload(testKey, dataFileGroup).get()).isTrue(); verifyAddGroupForDownloadWritesMetadata(testKey, dataFileGroup, CURRENT_TIMESTAMP); + + assertLoggedNewConfigs( + fakeEventLogger, createFileGroupDetails(dataFileGroup).clearFileCount().build(), null); } @Test public void testAddGroupForDownload_groupUpdated_whenVariantChanges() throws Exception { + FakeEventLogger fakeEventLogger = new FakeEventLogger(); + resetFileGroupManager(fakeEventLogger, fileGroupsMetadata, sharedFileManager); + DataFileGroupInternal dataFileGroup = MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 2); assertThat(fileGroupManager.addGroupForDownload(testKey, dataFileGroup).get()).isTrue(); verifyAddGroupForDownloadWritesMetadata(testKey, dataFileGroup, CURRENT_TIMESTAMP); + // Reset to clear events before next add group call + fakeEventLogger.reset(); + // Update the file id and see that the group gets updated in the pending groups list. dataFileGroup = dataFileGroup.toBuilder().setVariantId("some-different-variant").build(); assertThat(fileGroupManager.addGroupForDownload(testKey, dataFileGroup).get()).isTrue(); verifyAddGroupForDownloadWritesMetadata(testKey, dataFileGroup, CURRENT_TIMESTAMP); + + assertLoggedNewConfigs( + fakeEventLogger, createFileGroupDetails(dataFileGroup).clearFileCount().build(), null); } @Test @@ -422,6 +506,8 @@ public class FileGroupManagerTest { // Send the exact same group as the downloaded group, and check that it is considered duplicate. assertThat(fileGroupManager.addGroupForDownload(testKey, dataFileGroup).get()).isFalse(); + + verifyNoInteractions(mockLogger); } @Test @@ -451,15 +537,21 @@ public class FileGroupManagerTest { assertThat(fileGroupManager.addGroupForDownload(testKey, firstGroup).get()).isTrue(); verifyAddGroupForDownloadWritesMetadata(testKey, firstGroup, CURRENT_TIMESTAMP); + verify(mockLogger) + .logNewConfigReceived(createFileGroupDetails(firstGroup).clearFileCount().build(), null); + reset(mockLogger); + // Create a second group that is identical except for one different file id. DataFileGroupInternal.Builder secondGroup = MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 2).toBuilder(); secondGroup.setFile(0, secondGroup.getFile(0).toBuilder().setFileId("file2")); writeDownloadedFileGroup(testKey, secondGroup.build()); - // Send the same group as downloaded group, and check that it is not considered duplicate. + // Send the updated group, and check that it is not considered duplicate. assertThat(fileGroupManager.addGroupForDownload(testKey, secondGroup.build()).get()).isTrue(); verifyAddGroupForDownloadWritesMetadata(testKey, secondGroup.build(), CURRENT_TIMESTAMP); + verify(mockLogger) + .logNewConfigReceived(createFileGroupDetails(firstGroup).clearFileCount().build(), null); } @Test @@ -483,6 +575,9 @@ public class FileGroupManagerTest { // Verify that we tried to subscribe to only the first 2 files. assertThat(fileCaptor.getAllValues()).containsExactly(groupKeys[0], groupKeys[1]); + + verify(mockLogger) + .logNewConfigReceived(createFileGroupDetails(dataFileGroup).clearFileCount().build(), null); } @Test @@ -506,6 +601,9 @@ public class FileGroupManagerTest { // Verify that we tried to subscribe to only the first file. assertThat(fileCaptor.getAllValues()).containsExactly(groupKeys[0]); + + verify(mockLogger) + .logNewConfigReceived(createFileGroupDetails(dataFileGroup).clearFileCount().build(), null); } @Test @@ -550,6 +648,14 @@ public class FileGroupManagerTest { assertThrows( UninstalledAppException.class, () -> fileGroupManager.addGroupForDownload(uninstalledAppKey, dataFileGroup)); + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); + verifyNoMoreInteractions(mockLogger); } @Test @@ -566,6 +672,14 @@ public class FileGroupManagerTest { assertThrows( ExpiredFileGroupException.class, () -> fileGroupManager.addGroupForDownload(testKey, dataFileGroup)); + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); + verifyNoMoreInteractions(mockLogger); } @Test @@ -582,6 +696,14 @@ public class FileGroupManagerTest { assertThrows( ExpiredFileGroupException.class, () -> fileGroupManager.addGroupForDownload(testKey, dataFileGroup)); + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); + verifyNoMoreInteractions(mockLogger); } @Test @@ -604,6 +726,8 @@ public class FileGroupManagerTest { assertThat(sharedFileManager.getSharedFile(groupKeys[0]).get()).isNotNull(); assertThat(sharedFileManager.getSharedFile(groupKeys[1]).get()).isNotNull(); + verify(mockLogger) + .logNewConfigReceived(createFileGroupDetails(dataFileGroup).clearFileCount().build(), null); } @Test @@ -628,6 +752,9 @@ public class FileGroupManagerTest { assertThat(sharedFileManager.getSharedFile(groupKeys[0]).get()).isNotNull(); assertThat(sharedFileManager.getSharedFile(groupKeys[1]).get()).isNotNull(); + verify(mockLogger) + .logNewConfigReceived( + createFileGroupDetails(dataFileGroup.build()).clearFileCount().build(), null); } @Test @@ -670,6 +797,7 @@ public class FileGroupManagerTest { @Test public void testAddGroupForDownload_delayedDownload() throws Exception { flags.enableDelayedDownload = Optional.of(true); + // Create 2 groups, one of which requires device side activation. DataFileGroupInternal fileGroup1 = MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 2).toBuilder() @@ -825,6 +953,9 @@ public class FileGroupManagerTest { assertThat(fileGroupsMetadata.getAllStaleGroups().get()).isEmpty(); assertThat(fileGroupsMetadata.getAllGroupKeys().get()).isEmpty(); + + // There is no pending file group, so no call to clearSyncReasons. + verifyNoInteractions(mockLogger); } @Test @@ -873,7 +1004,7 @@ public class FileGroupManagerTest { newFileKey1.getChecksum(), mockSilentFeedback, /* instanceId= */ Optional.absent(), - /* androidShared = */ false); + /* androidShared= */ false); Uri pendingFileUri2 = DirectoryUtil.getOnDeviceUri( context, @@ -882,10 +1013,12 @@ public class FileGroupManagerTest { newFileKey2.getChecksum(), mockSilentFeedback, /* instanceId= */ Optional.absent(), - /* androidShared = */ false); + /* androidShared= */ false); + + verify(mockDownloader).stopDownloading(newFileKey1.getChecksum(), pendingFileUri1); + verify(mockDownloader).stopDownloading(newFileKey2.getChecksum(), pendingFileUri2); - verify(mockDownloader).stopDownloading(pendingFileUri1); - verify(mockDownloader).stopDownloading(pendingFileUri2); + verifyNoInteractions(mockLogger); } @Test @@ -930,7 +1063,9 @@ public class FileGroupManagerTest { .build()) .build()); - verify(mockDownloader, never()).stopDownloading(any(Uri.class)); + verify(mockDownloader, never()).stopDownloading(any(String.class), any(Uri.class)); + + verifyNoInteractions(mockLogger); } @Test @@ -996,10 +1131,12 @@ public class FileGroupManagerTest { registeredFileKey.getChecksum(), mockSilentFeedback, /* instanceId= */ Optional.absent(), - /* androidShared = */ false); + /* androidShared= */ false); // Only called once to stop download of pending file. - verify(mockDownloader).stopDownloading(pendingFileUri); + verify(mockDownloader).stopDownloading(registeredFileKey.getChecksum(), pendingFileUri); + + verifyNoInteractions(mockLogger); } @Test @@ -1053,7 +1190,7 @@ public class FileGroupManagerTest { assertThat(fileGroupsMetadata.getAllStaleGroups().get()).isEmpty(); // Downloaded group is still available. assertThat(fileGroupsMetadata.getAllFreshGroups().get()) - .containsExactly(Pair.create(downloadedGroupKey, downloadedFileGroup)); + .containsExactly(GroupKeyAndGroup.create(downloadedGroupKey, downloadedFileGroup)); Uri pendingFileUri = DirectoryUtil.getOnDeviceUri( @@ -1063,10 +1200,12 @@ public class FileGroupManagerTest { registeredFileKey.getChecksum(), mockSilentFeedback, /* instanceId= */ Optional.absent(), - /* androidShared = */ false); + /* androidShared= */ false); // Only called once to stop download of pending file. - verify(mockDownloader).stopDownloading(pendingFileUri); + verify(mockDownloader).stopDownloading(registeredFileKey.getChecksum(), pendingFileUri); + + verifyNoInteractions(mockLogger); } @Test @@ -1118,8 +1257,8 @@ public class FileGroupManagerTest { ImmutableList.of(FileStatus.DOWNLOAD_IN_PROGRESS, FileStatus.DOWNLOAD_IN_PROGRESS)); assertThat(fileGroupsMetadata.getAllFreshGroups().get()) .containsExactly( - new Pair<GroupKey, DataFileGroupInternal>(pendingGroupKey, pendingFileGroup), - new Pair<GroupKey, DataFileGroupInternal>(pendingGroupKey2, pendingFileGroup2)); + GroupKeyAndGroup.create(pendingGroupKey, pendingFileGroup), + GroupKeyAndGroup.create(pendingGroupKey2, pendingFileGroup2)); fileGroupManager.removeFileGroup(groupKey, /* pendingOnly= */ false).get(); @@ -1128,10 +1267,11 @@ public class FileGroupManagerTest { assertThat(readPendingFileGroup(downloadedGroupKey)).isNull(); assertThat(fileGroupsMetadata.getAllStaleGroups().get()).isEmpty(); assertThat(fileGroupsMetadata.getAllFreshGroups().get()) - .containsExactly( - new Pair<GroupKey, DataFileGroupInternal>(pendingGroupKey2, pendingFileGroup2)); + .containsExactly(GroupKeyAndGroup.create(pendingGroupKey2, pendingFileGroup2)); - verify(mockDownloader, never()).stopDownloading(any(Uri.class)); + verify(mockDownloader, never()).stopDownloading(any(String.class), any(Uri.class)); + + verifyNoInteractions(mockLogger); } @Test @@ -1177,6 +1317,8 @@ public class FileGroupManagerTest { verify(mockFileGroupsMetadata).remove(pendingGroupKey); verify(mockFileGroupsMetadata).remove(downloadedGroupKey); verify(mockFileGroupsMetadata, never()).addStaleGroup(any(DataFileGroupInternal.class)); + + verify(mockLogger).logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); } @Test @@ -1196,7 +1338,7 @@ public class FileGroupManagerTest { writePendingFileGroup(testKey, sideloadedGroup); writeDownloadedFileGroup(testKey, sideloadedGroup); - fileGroupManager.removeFileGroup(testKey, /* pendingOnly = */ false).get(); + fileGroupManager.removeFileGroup(testKey, /* pendingOnly= */ false).get(); assertThat(readPendingFileGroup(testKey)).isNull(); assertThat(readDownloadedFileGroup(testKey)).isNull(); @@ -1238,28 +1380,28 @@ public class FileGroupManagerTest { { // Perfrom removal once and check that the default group gets removed - fileGroupManager.removeFileGroup(defaultGroupKey, /* pendingOnly = */ false).get(); + fileGroupManager.removeFileGroup(defaultGroupKey, /* pendingOnly= */ false).get(); assertThat(fileGroupsMetadata.getAllGroupKeys().get()) .comparingElementsUsing(GROUP_KEY_TO_VARIANT) .containsExactly("en", "fr"); assertThat(fileGroupsMetadata.getAllFreshGroups().get()) - .comparingElementsUsing(KEY_GROUP_PAIR_TO_VARIANT_PAIR) - .containsExactly(Pair.create("en", "en"), Pair.create("fr", "fr")); + .comparingElementsUsing(KEY_GROUP_PAIR_TO_VARIANT) + .containsExactly("en", "fr"); assertThat(sharedFilesMetadata.getAllFileKeys().get()).hasSize(1); } { // Perform remove again and verify that there is no change in state - fileGroupManager.removeFileGroup(defaultGroupKey, /* pendingOnly = */ false).get(); + fileGroupManager.removeFileGroup(defaultGroupKey, /* pendingOnly= */ false).get(); assertThat(fileGroupsMetadata.getAllGroupKeys().get()) .comparingElementsUsing(GROUP_KEY_TO_VARIANT) .containsExactly("en", "fr"); assertThat(fileGroupsMetadata.getAllFreshGroups().get()) - .comparingElementsUsing(KEY_GROUP_PAIR_TO_VARIANT_PAIR) - .containsExactly(Pair.create("en", "en"), Pair.create("fr", "fr")); + .comparingElementsUsing(KEY_GROUP_PAIR_TO_VARIANT) + .containsExactly("en", "fr"); assertThat(sharedFilesMetadata.getAllFileKeys().get()).hasSize(1); } @@ -1300,28 +1442,28 @@ public class FileGroupManagerTest { { // Perfrom removal once and check that the en group gets removed - fileGroupManager.removeFileGroup(enGroupKey, /* pendingOnly = */ false).get(); + fileGroupManager.removeFileGroup(enGroupKey, /* pendingOnly= */ false).get(); assertThat(fileGroupsMetadata.getAllGroupKeys().get()) .comparingElementsUsing(GROUP_KEY_TO_VARIANT) .containsExactly("", "fr"); assertThat(fileGroupsMetadata.getAllFreshGroups().get()) - .comparingElementsUsing(KEY_GROUP_PAIR_TO_VARIANT_PAIR) - .containsExactly(Pair.create("", ""), Pair.create("fr", "fr")); + .comparingElementsUsing(KEY_GROUP_PAIR_TO_VARIANT) + .containsExactly("", "fr"); assertThat(sharedFilesMetadata.getAllFileKeys().get()).hasSize(1); } { // Perform remove again and verify that there is no change in state - fileGroupManager.removeFileGroup(enGroupKey, /* pendingOnly = */ false).get(); + fileGroupManager.removeFileGroup(enGroupKey, /* pendingOnly= */ false).get(); assertThat(fileGroupsMetadata.getAllGroupKeys().get()) .comparingElementsUsing(GROUP_KEY_TO_VARIANT) .containsExactly("", "fr"); assertThat(fileGroupsMetadata.getAllFreshGroups().get()) - .comparingElementsUsing(KEY_GROUP_PAIR_TO_VARIANT_PAIR) - .containsExactly(Pair.create("", ""), Pair.create("fr", "fr")); + .comparingElementsUsing(KEY_GROUP_PAIR_TO_VARIANT) + .containsExactly("", "fr"); assertThat(sharedFilesMetadata.getAllFileKeys().get()).hasSize(1); } @@ -1381,7 +1523,7 @@ public class FileGroupManagerTest { assertThat(fileGroupsMetadata.getAllFreshGroups().get()).hasSize(2); assertThat(fileGroupsMetadata.getAllStaleGroups().get()).isEmpty(); - verify(mockDownloader, times(0)).stopDownloading(any()); + verify(mockDownloader, times(0)).stopDownloading(any(), any()); } @Test @@ -1441,8 +1583,8 @@ public class FileGroupManagerTest { pendingGroupToRemove1.getFile(0).getFileId(), pendingFileKey1.getChecksum(), mockSilentFeedback, - /* instanceId = */ Optional.absent(), - /* androidShared = */ false); + /* instanceId= */ Optional.absent(), + /* androidShared= */ false); Uri pendingFileUri2 = DirectoryUtil.getOnDeviceUri( context, @@ -1450,8 +1592,8 @@ public class FileGroupManagerTest { pendingGroupToRemove2.getFile(0).getFileId(), pendingFileKey2.getChecksum(), mockSilentFeedback, - /* instanceId = */ Optional.absent(), - /* androidShared = */ false); + /* instanceId= */ Optional.absent(), + /* androidShared= */ false); Uri pendingFileUri3 = DirectoryUtil.getOnDeviceUri( context, @@ -1459,8 +1601,8 @@ public class FileGroupManagerTest { pendingGroupToKeep.getFile(0).getFileId(), pendingFileKey3.getChecksum(), mockSilentFeedback, - /* instanceId = */ Optional.absent(), - /* androidShared = */ false); + /* instanceId= */ Optional.absent(), + /* androidShared= */ false); // Assert that matching pending groups are removed assertThat(readPendingFileGroup(pendingGroupKeyToRemove1)).isNull(); @@ -1470,9 +1612,10 @@ public class FileGroupManagerTest { assertThat(fileGroupsMetadata.getAllFreshGroups().get()).hasSize(2); assertThat(fileGroupsMetadata.getAllStaleGroups().get()).isEmpty(); - verify(mockDownloader).stopDownloading(pendingFileUri1); - verify(mockDownloader).stopDownloading(pendingFileUri2); - verify(mockDownloader, times(0)).stopDownloading(pendingFileUri3); + verify(mockDownloader).stopDownloading(pendingFileKey1.getChecksum(), pendingFileUri1); + verify(mockDownloader).stopDownloading(pendingFileKey2.getChecksum(), pendingFileUri2); + verify(mockDownloader, times(0)) + .stopDownloading(pendingFileKey3.getChecksum(), pendingFileUri3); } @Test @@ -1529,8 +1672,8 @@ public class FileGroupManagerTest { pendingGroupToKeep.getFile(0).getFileId(), pendingFileKey1.getChecksum(), mockSilentFeedback, - /* instanceId = */ Optional.absent(), - /* androidShared = */ false); + /* instanceId= */ Optional.absent(), + /* androidShared= */ false); // Assert that matching pending groups are removed assertThat(readDownloadedFileGroup(downloadedGroupKeyToRemove1)).isNull(); @@ -1540,8 +1683,8 @@ public class FileGroupManagerTest { assertThat(fileGroupsMetadata.getAllFreshGroups().get()) .containsExactly( - Pair.create(downloadedGroupKeyToKeep, downloadedGroupToKeep), - Pair.create(pendingGroupKeyToKeep, pendingGroupToKeep)); + GroupKeyAndGroup.create(downloadedGroupKeyToKeep, downloadedGroupToKeep), + GroupKeyAndGroup.create(pendingGroupKeyToKeep, pendingGroupToKeep)); assertThat(fileGroupsMetadata.getAllStaleGroups().get()) .containsExactly( downloadedGroupToRemove1.toBuilder() @@ -1557,7 +1700,8 @@ public class FileGroupManagerTest { .build()) .build()); - verify(mockDownloader, times(0)).stopDownloading(pendingFileUri1); + verify(mockDownloader, times(0)) + .stopDownloading(pendingFileKey1.getChecksum(), pendingFileUri1); } @Test @@ -1622,8 +1766,8 @@ public class FileGroupManagerTest { pendingGroupToRemove1.getFile(0).getFileId(), pendingFileKey1.getChecksum(), mockSilentFeedback, - /* instanceId = */ Optional.absent(), - /* androidShared = */ false); + /* instanceId= */ Optional.absent(), + /* androidShared= */ false); Uri pendingFileUri2 = DirectoryUtil.getOnDeviceUri( context, @@ -1631,8 +1775,8 @@ public class FileGroupManagerTest { pendingGroupToRemove2.getFile(0).getFileId(), pendingFileKey2.getChecksum(), mockSilentFeedback, - /* instanceId = */ Optional.absent(), - /* androidShared = */ false); + /* instanceId= */ Optional.absent(), + /* androidShared= */ false); // Assert that matching pending groups are removed assertThat(readDownloadedFileGroup(downloadedGroupKeyToRemove1)).isNull(); @@ -1656,8 +1800,10 @@ public class FileGroupManagerTest { .build()) .build()); - verify(mockDownloader, times(1)).stopDownloading(pendingFileUri1); - verify(mockDownloader, times(1)).stopDownloading(pendingFileUri2); + verify(mockDownloader, times(1)) + .stopDownloading(pendingFileKey1.getChecksum(), pendingFileUri1); + verify(mockDownloader, times(1)) + .stopDownloading(pendingFileKey2.getChecksum(), pendingFileUri2); } @Test @@ -1708,17 +1854,21 @@ public class FileGroupManagerTest { assertThat(readPendingFileGroup(pendingGroupKeyToRemove2)).isNull(); assertThat(readPendingFileGroup(pendingGroupKeyToKeep)).isNotNull(); assertThat(fileGroupsMetadata.getAllFreshGroups().get()) - .containsExactly(Pair.create(pendingGroupKeyToKeep, pendingGroupToKeep)); + .containsExactly(GroupKeyAndGroup.create(pendingGroupKeyToKeep, pendingGroupToKeep)); // Get On Device Uris to check if file downloads were cancelled List<Uri> uncancelledFileUris = getOnDeviceUrisForFileGroup(pendingGroupToKeep); - verify(mockDownloader, times(0)).stopDownloading(uncancelledFileUris.get(0)); - verify(mockDownloader, times(0)).stopDownloading(uncancelledFileUris.get(1)); + verify(mockDownloader, times(0)).stopDownloading(any(), eq(uncancelledFileUris.get(0))); + verify(mockDownloader, times(0)).stopDownloading(any(), eq(uncancelledFileUris.get(1))); verify(mockDownloader, times(1)) - .stopDownloading(getOnDeviceUrisForFileGroup(pendingGroupToRemove1).get(0)); + .stopDownloading( + pendingGroupToRemove1.getFile(0).getChecksum(), + getOnDeviceUrisForFileGroup(pendingGroupToRemove1).get(0)); verify(mockDownloader, times(1)) - .stopDownloading(getOnDeviceUrisForFileGroup(pendingGroupToRemove2).get(0)); + .stopDownloading( + pendingGroupToRemove2.getFile(0).getChecksum(), + getOnDeviceUrisForFileGroup(pendingGroupToRemove2).get(0)); } @Test @@ -1753,6 +1903,7 @@ public class FileGroupManagerTest { verify(mockFileGroupsMetadata, times(0)).addStaleGroup(any()); verify(mockSharedFileManager, times(0)).cancelDownload(any()); + verify(mockLogger, times(1)).logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); verify(mockFileGroupsMetadata, times(1)).removeAllGroupsWithKeys(any()); List<GroupKey> attemptedRemoveKeys = groupKeysCaptor.getValue(); assertThat(attemptedRemoveKeys).containsExactly(pendingGroupKey); @@ -1801,6 +1952,7 @@ public class FileGroupManagerTest { verify(mockFileGroupsMetadata, times(0)).addStaleGroup(any()); verify(mockSharedFileManager, times(0)).cancelDownload(any()); + verify(mockLogger, times(1)).logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); verify(mockFileGroupsMetadata, times(2)).removeAllGroupsWithKeys(any()); List<List<GroupKey>> removeCallInvocations = groupKeysCaptor.getAllValues(); assertThat(removeCallInvocations.get(0)).containsExactly(pendingGroupKey); @@ -1852,6 +2004,7 @@ public class FileGroupManagerTest { verify(mockFileGroupsMetadata, times(1)).addStaleGroup(downloadedGroup); verify(mockSharedFileManager, times(0)).cancelDownload(any()); + verify(mockLogger, times(1)).logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); verify(mockFileGroupsMetadata, times(2)).removeAllGroupsWithKeys(any()); List<List<GroupKey>> removeCallInvocations = groupKeysCaptor.getAllValues(); assertThat(removeCallInvocations.get(0)).containsExactly(pendingGroupKey); @@ -1946,8 +2099,8 @@ public class FileGroupManagerTest { .comparingElementsUsing(GROUP_KEY_TO_VARIANT) .containsExactly("fr"); assertThat(fileGroupsMetadata.getAllFreshGroups().get()) - .comparingElementsUsing(KEY_GROUP_PAIR_TO_VARIANT_PAIR) - .containsExactly(Pair.create("fr", "fr")); + .comparingElementsUsing(KEY_GROUP_PAIR_TO_VARIANT) + .containsExactly("fr"); assertThat(sharedFilesMetadata.getAllFileKeys().get()).hasSize(1); } @@ -1960,8 +2113,8 @@ public class FileGroupManagerTest { .comparingElementsUsing(GROUP_KEY_TO_VARIANT) .containsExactly("fr"); assertThat(fileGroupsMetadata.getAllFreshGroups().get()) - .comparingElementsUsing(KEY_GROUP_PAIR_TO_VARIANT_PAIR) - .containsExactly(Pair.create("fr", "fr")); + .comparingElementsUsing(KEY_GROUP_PAIR_TO_VARIANT) + .containsExactly("fr"); assertThat(sharedFilesMetadata.getAllFileKeys().get()).hasSize(1); } @@ -2053,6 +2206,7 @@ public class FileGroupManagerTest { public void testSetGroupActivation_deactivationRemovesGroupsRequiringActivation() throws Exception { flags.enableDelayedDownload = Optional.of(true); + // Create 2 groups, one of which requires device side activation. DataFileGroupInternal.Builder fileGroup1 = MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 2).toBuilder(); @@ -2114,11 +2268,11 @@ public class FileGroupManagerTest { fileGroupManager .importFilesIntoFileGroup( groupKey, - /* buildId = */ 0, - /* variantId = */ "", + /* buildId= */ 0, + /* variantId= */ "", updatedDataFileList, - /* inlineFileMap = */ ImmutableMap.of(), - /* customPropertyOptional = */ Optional.absent(), + /* inlineFileMap= */ ImmutableMap.of(), + /* customPropertyOptional= */ Optional.absent(), noCustomValidation()) .get()); @@ -2153,11 +2307,11 @@ public class FileGroupManagerTest { fileGroupManager .importFilesIntoFileGroup( groupKey, - /* buildId = */ 1, - /* variantId = */ "", - /* updatedDataFileList = */ ImmutableList.of(), - /* inlineFileMap = */ ImmutableMap.of(), - /* customPropertyOptional = */ Optional.absent(), + /* buildId= */ 1, + /* variantId= */ "", + /* updatedDataFileList= */ ImmutableList.of(), + /* inlineFileMap= */ ImmutableMap.of(), + /* customPropertyOptional= */ Optional.absent(), noCustomValidation()) .get()); @@ -2193,11 +2347,11 @@ public class FileGroupManagerTest { fileGroupManager .importFilesIntoFileGroup( groupKey, - /* buildId = */ 0, - /* variantId = */ "testvariant", - /* updatedDataFileList = */ ImmutableList.of(), - /* inlineFileMap = */ ImmutableMap.of(), - /* customPropertyOptional = */ Optional.absent(), + /* buildId= */ 0, + /* variantId= */ "testvariant", + /* updatedDataFileList= */ ImmutableList.of(), + /* inlineFileMap= */ ImmutableMap.of(), + /* customPropertyOptional= */ Optional.absent(), noCustomValidation()) .get()); @@ -2237,11 +2391,11 @@ public class FileGroupManagerTest { fileGroupManager .importFilesIntoFileGroup( groupKey, - /* buildId = */ 3, - /* variantId = */ "testvariant", - /* updatedDataFileList = */ ImmutableList.of(), - /* inlineFileMap = */ ImmutableMap.of(), - /* customPropertyOptional = */ Optional.of(customProperty), + /* buildId= */ 3, + /* variantId= */ "testvariant", + /* updatedDataFileList= */ ImmutableList.of(), + /* inlineFileMap= */ ImmutableMap.of(), + /* customPropertyOptional= */ Optional.of(customProperty), noCustomValidation()) .get()); @@ -2281,11 +2435,11 @@ public class FileGroupManagerTest { fileGroupManager .importFilesIntoFileGroup( groupKey, - /* buildId = */ 1, - /* variantId = */ "testvariant3", - /* updatedDataFileList = */ ImmutableList.of(), - /* inlineFileMap = */ ImmutableMap.of(), - /* customPropertyOptional = */ Optional.of(customProperty), + /* buildId= */ 1, + /* variantId= */ "testvariant3", + /* updatedDataFileList= */ ImmutableList.of(), + /* inlineFileMap= */ ImmutableMap.of(), + /* customPropertyOptional= */ Optional.of(customProperty), noCustomValidation()) .get()); @@ -2335,11 +2489,11 @@ public class FileGroupManagerTest { fileGroupManager .importFilesIntoFileGroup( groupKey, - /* buildId = */ 1, - /* variantId = */ "testvariant", - /* updatedDataFileList = */ ImmutableList.of(), - /* inlineFileMap = */ ImmutableMap.of(), - /* customPropertyOptional = */ Optional.of(mismatchedCustomProperty), + /* buildId= */ 1, + /* variantId= */ "testvariant", + /* updatedDataFileList= */ ImmutableList.of(), + /* inlineFileMap= */ ImmutableMap.of(), + /* customPropertyOptional= */ Optional.of(mismatchedCustomProperty), noCustomValidation()) .get()); @@ -2386,11 +2540,11 @@ public class FileGroupManagerTest { fileGroupManager .importFilesIntoFileGroup( groupKey, - /* buildId = */ 1, - /* variantId = */ "testvariant", - /* updatedDataFileList = */ ImmutableList.of(), - /* inlineFileMap = */ ImmutableMap.of(), - /* customPropertyOptional = */ Optional.absent(), + /* buildId= */ 1, + /* variantId= */ "testvariant", + /* updatedDataFileList= */ ImmutableList.of(), + /* inlineFileMap= */ ImmutableMap.of(), + /* customPropertyOptional= */ Optional.absent(), noCustomValidation()) .get()); @@ -2441,11 +2595,11 @@ public class FileGroupManagerTest { fileGroupManager .importFilesIntoFileGroup( groupKey, - /* buildId = */ 0, - /* variantId = */ "", - /* updatedDataFileList = */ updatedDataFileList, - /* inlineFileMap = */ ImmutableMap.of(), - /* customPropertyOptional = */ Optional.absent(), + /* buildId= */ 0, + /* variantId= */ "", + /* updatedDataFileList= */ updatedDataFileList, + /* inlineFileMap= */ ImmutableMap.of(), + /* customPropertyOptional= */ Optional.absent(), noCustomValidation()) .get()); assertThat(ex).hasCauseThat().isInstanceOf(DownloadException.class); @@ -2483,11 +2637,11 @@ public class FileGroupManagerTest { fileGroupManager .importFilesIntoFileGroup( groupKey, - /* buildId = */ 0, - /* variantId = */ "", - /* updatedDataFileList = */ ImmutableList.of(), + /* buildId= */ 0, + /* variantId= */ "", + /* updatedDataFileList= */ ImmutableList.of(), inlineFileMap, - /* customPropertyOptional = */ Optional.absent(), + /* customPropertyOptional= */ Optional.absent(), noCustomValidation()) .get(); @@ -2515,11 +2669,11 @@ public class FileGroupManagerTest { fileGroupManager .importFilesIntoFileGroup( groupKey, - /* buildId = */ 0, - /* variantId = */ "", - /* updatedDataFileList = */ ImmutableList.of(), - /* inlineFileMap = */ ImmutableMap.of(), - /* customPropertyOptional = */ Optional.absent(), + /* buildId= */ 0, + /* variantId= */ "", + /* updatedDataFileList= */ ImmutableList.of(), + /* inlineFileMap= */ ImmutableMap.of(), + /* customPropertyOptional= */ Optional.absent(), noCustomValidation()) .get(); @@ -2571,11 +2725,11 @@ public class FileGroupManagerTest { fileGroupManager .importFilesIntoFileGroup( groupKey, - /* buildId = */ 0, - /* variantId = */ "", + /* buildId= */ 0, + /* variantId= */ "", updatedDataFileList, inlineFileMap, - /* customPropertyOptional = */ Optional.absent(), + /* customPropertyOptional= */ Optional.absent(), noCustomValidation()) .get(); @@ -2627,11 +2781,11 @@ public class FileGroupManagerTest { fileGroupManager .importFilesIntoFileGroup( groupKey, - /* buildId = */ 0, - /* variantId = */ "", + /* buildId= */ 0, + /* variantId= */ "", updatedDataFileList, inlineFileMap, - /* customPropertyOptional = */ Optional.absent(), + /* customPropertyOptional= */ Optional.absent(), noCustomValidation()) .get(); @@ -2696,11 +2850,11 @@ public class FileGroupManagerTest { fileGroupManager .importFilesIntoFileGroup( groupKey, - /* buildId = */ 10, - /* variantId = */ "", + /* buildId= */ 10, + /* variantId= */ "", updatedDataFileList, inlineFileMap, - /* customPropertyOptional = */ Optional.absent(), + /* customPropertyOptional= */ Optional.absent(), noCustomValidation()) .get(); @@ -2770,11 +2924,11 @@ public class FileGroupManagerTest { fileGroupManager .importFilesIntoFileGroup( groupKey, - /* buildId = */ 0, - /* variantId = */ "", + /* buildId= */ 0, + /* variantId= */ "", updatedDataFileList, inlineFileMap, - /* customPropertyOptional = */ Optional.absent(), + /* customPropertyOptional= */ Optional.absent(), noCustomValidation()) .get(); @@ -2834,11 +2988,11 @@ public class FileGroupManagerTest { fileGroupManager .importFilesIntoFileGroup( groupKey, - /* buildId = */ 0, - /* variantId = */ "", - /* updatedDataFileList = */ ImmutableList.of(), - /* inlineFileMap = */ ImmutableMap.of("inline-file", testFileSource), - /* customPropertyOptional = */ Optional.absent(), + /* buildId= */ 0, + /* variantId= */ "", + /* updatedDataFileList= */ ImmutableList.of(), + /* inlineFileMap= */ ImmutableMap.of("inline-file", testFileSource), + /* customPropertyOptional= */ Optional.absent(), noCustomValidation()) .get(); @@ -2886,11 +3040,11 @@ public class FileGroupManagerTest { fileGroupManager .importFilesIntoFileGroup( groupKey, - /* buildId = */ 0, - /* variantId = */ "", - /* updatedDataFileList = */ ImmutableList.of(), - /* inlineFileMap = */ ImmutableMap.of("inline-file", testFileSource), - /* customPropertyOptional = */ Optional.absent(), + /* buildId= */ 0, + /* variantId= */ "", + /* updatedDataFileList= */ ImmutableList.of(), + /* inlineFileMap= */ ImmutableMap.of("inline-file", testFileSource), + /* customPropertyOptional= */ Optional.absent(), noCustomValidation()) .get(); @@ -2926,11 +3080,11 @@ public class FileGroupManagerTest { fileGroupManager .importFilesIntoFileGroup( groupKey, - /* buildId = */ 0, - /* variantId = */ "", - /* updatedDataFileList = */ ImmutableList.of(), - /* inlineFileMap = */ ImmutableMap.of(), - /* customPropertyOptional = */ Optional.absent(), + /* buildId= */ 0, + /* variantId= */ "", + /* updatedDataFileList= */ ImmutableList.of(), + /* inlineFileMap= */ ImmutableMap.of(), + /* customPropertyOptional= */ Optional.absent(), noCustomValidation()) .get()); assertThat(ex).hasCauseThat().isInstanceOf(AggregateException.class); @@ -3006,12 +3160,12 @@ public class FileGroupManagerTest { fileGroupManager .importFilesIntoFileGroup( groupKey, - /* buildId = */ 0, - /* variantId = */ "", + /* buildId= */ 0, + /* variantId= */ "", updatedDataFileList, - /* inlineFileMap = */ ImmutableMap.of( + /* inlineFileMap= */ ImmutableMap.of( "inline-file-1", testFileSource1, "inline-file-2", testFileSource2), - /* customPropertyOptional = */ Optional.absent(), + /* customPropertyOptional= */ Optional.absent(), noCustomValidation()) .get()); @@ -3076,9 +3230,9 @@ public class FileGroupManagerTest { testKey, sideloadedGroup.getBuildId(), sideloadedGroup.getVariantId(), - /* updatedDataFileList = */ ImmutableList.of(), + /* updatedDataFileList= */ ImmutableList.of(), inlineFileMap, - /* customPropertyOptional = */ Optional.absent(), + /* customPropertyOptional= */ Optional.absent(), noCustomValidation()) .get(); @@ -3092,9 +3246,9 @@ public class FileGroupManagerTest { DataFileGroupInternal fileGroup = createDataFileGroup( TEST_GROUP, - /*fileCount=*/ 2, - /*downloadAttemptCount=*/ 3, - /*newFilesReceivedTimestamp=*/ testClock.currentTimeMillis() - 500L); + /* fileCount= */ 2, + /* downloadAttemptCount= */ 3, + /* newFilesReceivedTimestamp= */ testClock.currentTimeMillis() - 500L); ExtraHttpHeader extraHttpHeader = ExtraHttpHeader.newBuilder().setKey("user-agent").setValue("mdd-downloader").build(); @@ -3121,6 +3275,27 @@ public class FileGroupManagerTest { // Verify that downloaded key is written into metadata if download is complete. assertThat(readDownloadedFileGroup(testKey)).isNotNull(); + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP, + 0, + /* buildId= */ 0, + /* variantId= */ ""); + verify(mockLogger) + .logMddDownloadResult( + MddDownloadResult.Code.SUCCESS, + createFileGroupDetails(fileGroup) + .setOwnerPackage(context.getPackageName()) + .clearFileCount() + .build()); + verify(mockLogger) + .logMddDownloadLatency( + createFileGroupDetails(fileGroup).build(), + createMddDownloadLatency( + /* downloadAttemptCount= */ 4, + /* downloadLatencyMs= */ 0L, + /* totalLatencyMs= */ 500L)); } @Test @@ -3129,9 +3304,9 @@ public class FileGroupManagerTest { DataFileGroupInternal fileGroup = createDataFileGroup( TEST_GROUP, - /*fileCount=*/ 2, - /*downloadAttemptCount=*/ 3, - /*newFilesReceivedTimestamp=*/ testClock.currentTimeMillis() - 500L); + /* fileCount= */ 2, + /* downloadAttemptCount= */ 3, + /* newFilesReceivedTimestamp= */ testClock.currentTimeMillis() - 500L); ExtraHttpHeader extraHttpHeader = ExtraHttpHeader.newBuilder().setKey("user-agent").setValue("mdd-downloader").build(); @@ -3164,6 +3339,30 @@ public class FileGroupManagerTest { // Verify that pending key was removed. This will ensure the files are eligible for garbage // collection. assertThat(readPendingFileGroup(testKey)).isNull(); + + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); + + ArgumentCaptor<MddDownloadResult.Code> resultCodeCaptor = + ArgumentCaptor.forClass(MddDownloadResult.Code.class); + ArgumentCaptor<DataDownloadFileGroupStats> groupDetailsCaptor = + ArgumentCaptor.forClass(DataDownloadFileGroupStats.class); + verify(mockLogger) + .logMddDownloadResult(resultCodeCaptor.capture(), groupDetailsCaptor.capture()); + + // Also clearing the file group version number becaused it ends up not being attached since + // the pending group was removed. + DataDownloadFileGroupStats expectedGroupDetails = + createFileGroupDetails(fileGroup).clearFileCount().clearFileGroupVersionNumber().build(); + + assertThat(resultCodeCaptor.getAllValues()) + .containsExactly(MddDownloadResult.Code.CUSTOM_FILEGROUP_VALIDATION_FAILED); + assertThat(groupDetailsCaptor.getAllValues()).containsExactly(expectedGroupDetails); } @Test @@ -3186,7 +3385,7 @@ public class FileGroupManagerTest { writeSharedFiles( sharedFilesMetadata, fileGroup, - ImmutableList.of(FileStatus.DOWNLOAD_IN_PROGRESS, FileStatus.DOWNLOAD_IN_PROGRESS)); + ImmutableList.of(FileStatus.SUBSCRIBED, FileStatus.SUBSCRIBED)); // First file failed. Uri failingFileUri = @@ -3230,6 +3429,28 @@ public class FileGroupManagerTest { // Verify that the pending group is not changed from pending to downloaded. assertThat(readDownloadedFileGroup(testKey)).isNull(); + + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 10, + /* variantId= */ "test-variant"); + + ArgumentCaptor<MddDownloadResult.Code> resultCodeCaptor = + ArgumentCaptor.forClass(MddDownloadResult.Code.class); + ArgumentCaptor<DataDownloadFileGroupStats> groupDetailsCaptor = + ArgumentCaptor.forClass(DataDownloadFileGroupStats.class); + verify(mockLogger) + .logMddDownloadResult(resultCodeCaptor.capture(), groupDetailsCaptor.capture()); + + DataDownloadFileGroupStats expectedGroupDetails = + createFileGroupDetails(fileGroup).clearFileCount().build(); + + assertThat(resultCodeCaptor.getAllValues()) + .containsExactly(MddDownloadResult.Code.LOW_DISK_ERROR); + assertThat(groupDetailsCaptor.getAllValues()).containsExactly(expectedGroupDetails); } @Test @@ -3248,10 +3469,7 @@ public class FileGroupManagerTest { writeSharedFiles( sharedFilesMetadata, fileGroup, - ImmutableList.of( - FileStatus.DOWNLOAD_IN_PROGRESS, - FileStatus.DOWNLOAD_IN_PROGRESS, - FileStatus.DOWNLOAD_IN_PROGRESS)); + ImmutableList.of(FileStatus.SUBSCRIBED, FileStatus.SUBSCRIBED, FileStatus.SUBSCRIBED)); // First file succeeded. Uri succeedingFileUri = @@ -3310,6 +3528,31 @@ public class FileGroupManagerTest { // Verify that the pending group is not changed from pending to downloaded. assertThat(readDownloadedFileGroup(testKey)).isNull(); + + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); + + ArgumentCaptor<MddDownloadResult.Code> resultCodeCaptor = + ArgumentCaptor.forClass(MddDownloadResult.Code.class); + ArgumentCaptor<DataDownloadFileGroupStats> groupDetailsCaptor = + ArgumentCaptor.forClass(DataDownloadFileGroupStats.class); + verify(mockLogger, times(2)) + .logMddDownloadResult(resultCodeCaptor.capture(), groupDetailsCaptor.capture()); + + DataDownloadFileGroupStats expectedGroupDetails = + createFileGroupDetails(fileGroup).clearFileCount().build(); + + assertThat(resultCodeCaptor.getAllValues()) + .containsExactly( + MddDownloadResult.Code.DOWNLOAD_TRANSFORM_IO_ERROR, + MddDownloadResult.Code.ANDROID_DOWNLOADER_HTTP_ERROR); + assertThat(groupDetailsCaptor.getAllValues()) + .containsExactly(expectedGroupDetails, expectedGroupDetails); } @Test @@ -3327,7 +3570,7 @@ public class FileGroupManagerTest { writeSharedFiles( sharedFilesMetadata, fileGroup, - ImmutableList.of(FileStatus.DOWNLOAD_IN_PROGRESS, FileStatus.DOWNLOAD_COMPLETE)); + ImmutableList.of(FileStatus.SUBSCRIBED, FileStatus.DOWNLOAD_COMPLETE)); // First file failed. Uri failingFileUri = @@ -3341,7 +3584,7 @@ public class FileGroupManagerTest { false); // The file status is set to DOWNLOAD_FAILED but the downloader returns an immediateVoidFuture. // An UNKNOWN_ERROR is logged. - fileDownloadFails(keys[0], failingFileUri, /* failureCode = */ null); + fileDownloadFails(keys[0], failingFileUri, /* failureCode= */ null); ListenableFuture<DataFileGroupInternal> downloadFuture = fileGroupManager.downloadFileGroup( @@ -3355,6 +3598,14 @@ public class FileGroupManagerTest { // Verify that the pending group is not changed from pending to downloaded. assertThat(readDownloadedFileGroup(testKey)).isNull(); + + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); } @Test @@ -3372,12 +3623,14 @@ public class FileGroupManagerTest { writeSharedFiles( sharedFilesMetadata, fileGroup, - ImmutableList.of(FileStatus.DOWNLOAD_COMPLETE, FileStatus.DOWNLOAD_IN_PROGRESS)); + ImmutableList.of(FileStatus.DOWNLOAD_COMPLETE, FileStatus.SUBSCRIBED)); when(mockDownloader.startDownloading( + any(String.class), any(GroupKey.class), anyInt(), anyLong(), + any(String.class), any(Uri.class), any(String.class), anyInt(), @@ -3399,6 +3652,21 @@ public class FileGroupManagerTest { // Verify that the pending group is not changed from pending to downloaded. assertThat(readDownloadedFileGroup(testKey)).isNull(); + + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); + verify(mockLogger) + .logMddDownloadResult( + MddDownloadResult.Code.UNKNOWN_ERROR, + createFileGroupDetails(fileGroup) + .setOwnerPackage(context.getPackageName()) + .clearFileCount() + .build()); } @Test @@ -3446,14 +3714,16 @@ public class FileGroupManagerTest { writeSharedFiles( sharedFilesMetadata, fileGroup, - ImmutableList.of(FileStatus.DOWNLOAD_IN_PROGRESS, FileStatus.DOWNLOAD_IN_PROGRESS)); + ImmutableList.of(FileStatus.SUBSCRIBED, FileStatus.SUBSCRIBED)); ArgumentCaptor<DownloadConditions> downloadConditionsCaptor = ArgumentCaptor.forClass(DownloadConditions.class); when(mockDownloader.startDownloading( + any(String.class), any(GroupKey.class), anyInt(), anyLong(), + any(String.class), any(Uri.class), any(String.class), anyInt(), @@ -3508,14 +3778,16 @@ public class FileGroupManagerTest { writeSharedFiles( sharedFilesMetadata, fileGroup, - ImmutableList.of(FileStatus.DOWNLOAD_IN_PROGRESS, FileStatus.DOWNLOAD_IN_PROGRESS)); + ImmutableList.of(FileStatus.SUBSCRIBED, FileStatus.SUBSCRIBED)); ArgumentCaptor<DownloadConditions> downloadConditionsCaptor = ArgumentCaptor.forClass(DownloadConditions.class); when(mockDownloader.startDownloading( + any(String.class), any(GroupKey.class), anyInt(), anyLong(), + any(String.class), any(Uri.class), any(String.class), anyInt(), @@ -3610,6 +3882,14 @@ public class FileGroupManagerTest { .isEqualTo(testClock.currentTimeMillis()); // Make sure that the download started count is accumulated. assertThat(bookkeeping.getDownloadStartedCount()).isEqualTo(1); + + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + fileGroup.getGroupName(), + fileGroup.getFileGroupVersionNumber(), + /* buildId= */ 0, + /* variantId= */ ""); } @Test @@ -3644,6 +3924,14 @@ public class FileGroupManagerTest { assertThat(bookkeeping.getGroupDownloadStartedTimestampInMillis()).isEqualTo(123456); // Make sure that the download started count is accumulated. assertThat(bookkeeping.getDownloadStartedCount()).isEqualTo(3); + + verify(mockLogger, never()) + .logEventSampled( + eq(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED), + any(String.class), + anyInt(), + anyLong(), + any(String.class)); } @Test @@ -3690,6 +3978,15 @@ public class FileGroupManagerTest { assertThat(bookkeeping.hasGroupDownloadStartedTimestampInMillis()).isTrue(); assertThat(bookkeeping.getGroupDownloadStartedTimestampInMillis()) .isEqualTo(testClock.currentTimeMillis()); + + verify(mockLogger, never()) + .logEventSampled( + eq(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED), + any(String.class), + anyInt(), + anyLong(), + any(String.class)); + verify(mockLogger).logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); } @Test @@ -3699,9 +3996,9 @@ public class FileGroupManagerTest { DataFileGroupInternal fileGroup = createDataFileGroup( TEST_GROUP, - /*fileCount=*/ 0, - /*downloadAttemptCount=*/ 3, - /*newFilesReceivedTimestamp=*/ testClock.currentTimeMillis() - 500L); + /* fileCount= */ 0, + /* downloadAttemptCount= */ 3, + /* newFilesReceivedTimestamp= */ testClock.currentTimeMillis() - 500L); ExtraHttpHeader extraHttpHeader = ExtraHttpHeader.newBuilder().setKey("user-agent").setValue("mdd-downloader").build(); @@ -3730,6 +4027,28 @@ public class FileGroupManagerTest { // Verify that downloaded key is written into metadata if download is complete. assertThat(readDownloadedFileGroup(testKey)).isNotNull(); + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); + verify(mockLogger) + .logMddDownloadResult( + MddDownloadResult.Code.SUCCESS, + createFileGroupDetails(fileGroup) + .setOwnerPackage(context.getPackageName()) + .clearFileCount() + .build()); + + verify(mockLogger) + .logMddDownloadLatency( + createFileGroupDetails(fileGroup).build(), + createMddDownloadLatency( + /* downloadAttemptCount= */ 4, + /* downloadLatencyMs= */ 0L, + /* totalLatencyMs= */ 500L)); // exists only called once in tryToShareBeforeDownload verify(mockBackend, never()).exists(any()); @@ -3780,6 +4099,13 @@ public class FileGroupManagerTest { // Verify that downloaded key is written into metadata if download is complete. assertThat(readDownloadedFileGroup(testKey)).isNotNull(); + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 10, + /* variantId= */ "test-variant"); verify(mockBackend, never()).exists(blobUri); verify(mockBackend, never()).openForWrite(blobUri); @@ -3789,6 +4115,11 @@ public class FileGroupManagerTest { assertThat(sharedFileManager.getSharedFile(keys[1]).get()).isEqualTo(file1); ArgumentCaptor<Void> mddAndroidSharingLogArgumentCaptor = ArgumentCaptor.forClass(Void.class); + + verify(mockLogger).logMddAndroidSharingLog(mddAndroidSharingLogArgumentCaptor.capture()); + Void mddAndroidSharingLog = null; + Void expectedLog = null; + assertThat(mddAndroidSharingLog).isEqualTo(expectedLog); } @Test @@ -3815,7 +4146,7 @@ public class FileGroupManagerTest { writeSharedFiles( sharedFilesMetadata, fileGroup, - ImmutableList.of(FileStatus.DOWNLOAD_IN_PROGRESS, FileStatus.DOWNLOAD_IN_PROGRESS)); + ImmutableList.of(FileStatus.SUBSCRIBED, FileStatus.SUBSCRIBED)); // File that can be shared DataFile file = fileGroup.getFile(0); @@ -3835,7 +4166,7 @@ public class FileGroupManagerTest { keys[0].getChecksum(), mockSilentFeedback, /* instanceId= */ Optional.absent(), - /* androidShared = */ false); + /* androidShared= */ false); fileDownloadSucceeds(keys[0], onDeviceuri); // Second file's download succeeds @@ -3847,7 +4178,7 @@ public class FileGroupManagerTest { keys[1].getChecksum(), mockSilentFeedback, /* instanceId= */ Optional.absent(), - /* androidShared = */ false); + /* androidShared= */ false); fileDownloadSucceeds(keys[1], onDeviceuri); fileGroupManager @@ -3860,6 +4191,14 @@ public class FileGroupManagerTest { // Verify that the downloaded group is still part of downloaded groups prefs. assertThat(readDownloadedFileGroup(testKey)).isNotNull(); + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); + verify(mockBackend).exists(blobUri); // openForWrite is called only once in tryToShareBeforeDownload for acquiring the lease. verify(mockBackend, never()).openForWrite(blobUri); @@ -3881,6 +4220,13 @@ public class FileGroupManagerTest { .build(); assertThat(sharedFileManager.getSharedFile(keys[0]).get()).isEqualTo(expectedSharedFile0); assertThat(sharedFileManager.getSharedFile(keys[1]).get()).isEqualTo(expectedSharedFile1); + + ArgumentCaptor<Void> mddAndroidSharingLogArgumentCaptor = ArgumentCaptor.forClass(Void.class); + + verify(mockLogger).logMddAndroidSharingLog(mddAndroidSharingLogArgumentCaptor.capture()); + Void mddAndroidSharingLog = null; + Void expectedLog = null; + assertThat(mddAndroidSharingLog).isEqualTo(expectedLog); } @Test @@ -3907,7 +4253,7 @@ public class FileGroupManagerTest { writeSharedFiles( sharedFilesMetadata, fileGroup, - ImmutableList.of(FileStatus.DOWNLOAD_IN_PROGRESS, FileStatus.DOWNLOAD_COMPLETE)); + ImmutableList.of(FileStatus.SUBSCRIBED, FileStatus.DOWNLOAD_COMPLETE)); // File that can be shared DataFile file = fileGroup.getFile(0); @@ -3928,13 +4274,15 @@ public class FileGroupManagerTest { keys[0].getChecksum(), mockSilentFeedback, /* instanceId= */ Optional.absent(), - /* androidShared = */ false); + /* androidShared= */ false); assertThat(fileStorage.exists(onDeviceuri)).isTrue(); when(mockDownloader.startDownloading( + any(String.class), any(GroupKey.class), anyInt(), anyLong(), + any(String.class), eq(onDeviceuri), any(String.class), anyInt(), @@ -3965,6 +4313,13 @@ public class FileGroupManagerTest { // Verify that downloaded key is written into metadata if download is complete. assertThat(readDownloadedFileGroup(testKey)).isNotNull(); + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); // exists called once in tryToShareBeforeDownload and once in tryToShareAfterDownload verify(mockBackend, times(2)).exists(blobUri); @@ -3989,10 +4344,15 @@ public class FileGroupManagerTest { assertThat(sharedFileManager.getSharedFile(keys[0]).get()).isEqualTo(expectedSharedFile0); assertThat(sharedFileManager.getSharedFile(keys[1]).get()).isEqualTo(expectedSharedFile1); - // tryToShareAfterDownload deletes the file - assertThat(fileStorage.exists(onDeviceuri)).isFalse(); + // Local copy has not been deleted. + assertThat(fileStorage.exists(onDeviceuri)).isTrue(); ArgumentCaptor<Void> mddAndroidSharingLogArgumentCaptor = ArgumentCaptor.forClass(Void.class); + + verify(mockLogger).logMddAndroidSharingLog(mddAndroidSharingLogArgumentCaptor.capture()); + Void mddAndroidSharingLog = null; + Void expectedLog = null; + assertThat(mddAndroidSharingLog).isEqualTo(expectedLog); } @Test @@ -4038,7 +4398,7 @@ public class FileGroupManagerTest { keys[0].getChecksum(), mockSilentFeedback, /* instanceId= */ Optional.absent(), - /* androidShared = */ false); + /* androidShared= */ false); assertThat(fileStorage.exists(onDeviceuri)).isTrue(); fileGroupManager @@ -4050,6 +4410,13 @@ public class FileGroupManagerTest { // Verify that downloaded key is written into metadata if download is complete. assertThat(readDownloadedFileGroup(testKey)).isNotNull(); + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); // exists only called once in tryToShareBeforeDownload verify(mockBackend).exists(blobUri); @@ -4078,6 +4445,11 @@ public class FileGroupManagerTest { onDeviceFile.delete(); ArgumentCaptor<Void> mddAndroidSharingLogArgumentCaptor = ArgumentCaptor.forClass(Void.class); + + verify(mockLogger).logMddAndroidSharingLog(mddAndroidSharingLogArgumentCaptor.capture()); + Void mddAndroidSharingLog = null; + Void expectedLog = null; + assertThat(mddAndroidSharingLog).isEqualTo(expectedLog); } @Test @@ -4103,7 +4475,7 @@ public class FileGroupManagerTest { writeSharedFiles( sharedFilesMetadata, fileGroup, - ImmutableList.of(FileStatus.DOWNLOAD_IN_PROGRESS, FileStatus.DOWNLOAD_COMPLETE)); + ImmutableList.of(FileStatus.SUBSCRIBED, FileStatus.DOWNLOAD_COMPLETE)); // File that can be copied to the blob storage DataFile file = fileGroup.getFile(0); @@ -4121,13 +4493,15 @@ public class FileGroupManagerTest { keys[0].getChecksum(), mockSilentFeedback, /* instanceId= */ Optional.absent(), - /* androidShared = */ false); + /* androidShared= */ false); assertThat(fileStorage.exists(onDeviceuri)).isTrue(); when(mockDownloader.startDownloading( + any(String.class), any(GroupKey.class), anyInt(), anyLong(), + any(String.class), eq(onDeviceuri), any(String.class), anyInt(), @@ -4158,6 +4532,13 @@ public class FileGroupManagerTest { // Verify that downloaded key is written into metadata if download is complete. assertThat(readDownloadedFileGroup(testKey)).isNotNull(); + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); // exists only called once in tryToShareBeforeDownload, once in tryToShareAfterDownload verify(mockBackend, times(2)).exists(blobUri); @@ -4181,10 +4562,15 @@ public class FileGroupManagerTest { assertThat(sharedFileManager.getSharedFile(keys[0]).get()).isEqualTo(expectedSharedFile0); assertThat(sharedFileManager.getSharedFile(keys[1]).get()).isEqualTo(expectedSharedFile1); - // File deleted after being copied to the blob storage. - assertThat(fileStorage.exists(onDeviceuri)).isFalse(); + // Local copy has not been deleted. + assertThat(fileStorage.exists(onDeviceuri)).isTrue(); ArgumentCaptor<Void> mddAndroidSharingLogArgumentCaptor = ArgumentCaptor.forClass(Void.class); + + verify(mockLogger).logMddAndroidSharingLog(mddAndroidSharingLogArgumentCaptor.capture()); + Void mddAndroidSharingLog = null; + Void expectedLog = null; + assertThat(mddAndroidSharingLog).isEqualTo(expectedLog); } @Test @@ -4205,8 +4591,7 @@ public class FileGroupManagerTest { NewFileKey[] keys = MddTestUtil.createFileKeysForDataFileGroupInternal(fileGroup); writePendingFileGroup(testKey, fileGroup); - writeSharedFiles( - sharedFilesMetadata, fileGroup, ImmutableList.of(FileStatus.DOWNLOAD_IN_PROGRESS)); + writeSharedFiles(sharedFilesMetadata, fileGroup, ImmutableList.of(FileStatus.SUBSCRIBED)); DataFile file = fileGroup.getFile(0); File onDeviceFile = simulateDownload(file, file.getFileId()); @@ -4218,7 +4603,7 @@ public class FileGroupManagerTest { keys[0].getChecksum(), mockSilentFeedback, /* instanceId= */ Optional.absent(), - /* androidShared = */ false); + /* androidShared= */ false); assertThat(fileStorage.exists(onDeviceuri)).isTrue(); fileDownloadSucceeds(keys[0], onDeviceuri); @@ -4232,6 +4617,13 @@ public class FileGroupManagerTest { // Verify that downloaded key is written into metadata if download is complete. assertThat(readDownloadedFileGroup(testKey)).isNotNull(); + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); verify(mockBackend, never()).exists(any()); verify(mockBackend, never()).openForWrite(any()); @@ -4301,6 +4693,33 @@ public class FileGroupManagerTest { expectedSharedFile0.toBuilder().setFileName(fileGroup.getFile(1).getFileId()).build(); assertThat(sharedFileManager.getSharedFile(keys[0]).get()).isEqualTo(expectedSharedFile0); assertThat(sharedFileManager.getSharedFile(keys[1]).get()).isEqualTo(expectedSharedFile1); + + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); + verify(mockLogger) + .logMddDownloadResult( + MddDownloadResult.Code.SUCCESS, + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(TEST_GROUP) + .setOwnerPackage(context.getPackageName()) + .setFileGroupVersionNumber(0) + .setBuildId(0) + .setVariantId("") + .build()); + + ArgumentCaptor<Void> mddAndroidSharingLogArgumentCaptor = ArgumentCaptor.forClass(Void.class); + + verify(mockLogger, times(2)) + .logMddAndroidSharingLog(mddAndroidSharingLogArgumentCaptor.capture()); + Void mddAndroidSharingLogBeforeAndAfterDownload = null; + assertThat(mddAndroidSharingLogArgumentCaptor.getAllValues()) + .containsExactly( + mddAndroidSharingLogBeforeAndAfterDownload, mddAndroidSharingLogBeforeAndAfterDownload); } @Test @@ -4331,7 +4750,7 @@ public class FileGroupManagerTest { SharedFile normalSharedFile = SharedFile.newBuilder() .setFileName(sideloadedGroup.getFile(1).getFileId()) - .setFileStatus(FileStatus.DOWNLOAD_IN_PROGRESS) + .setFileStatus(FileStatus.SUBSCRIBED) .build(); sharedFilesMetadata.write(normalFileKey, normalSharedFile).get(); @@ -4343,7 +4762,7 @@ public class FileGroupManagerTest { sideloadedGroup.getFile(1).getFileId(), sideloadedGroup.getFile(1).getChecksum(), mockSilentFeedback, - /* instanceId = */ Optional.absent(), + /* instanceId= */ Optional.absent(), false); fileDownloadSucceeds(normalFileKey, normalFileUri); @@ -4356,9 +4775,11 @@ public class FileGroupManagerTest { verify(mockDownloader) .startDownloading( + eq(sideloadedGroup.getFile(1).getChecksum()), eq(testKey), anyInt(), anyLong(), + any(String.class), eq(normalFileUri), eq(sideloadedGroup.getFile(1).getUrlToDownload()), anyInt(), @@ -4431,9 +4852,9 @@ public class FileGroupManagerTest { DataFileGroupInternal fileGroup1 = createDataFileGroup( TEST_GROUP, - /*fileCount=*/ 2, - /*downloadAttemptCount=*/ 7, - /*newFilesReceivedTimestamp=*/ testClock.currentTimeMillis() - 500L); + /* fileCount= */ 2, + /* downloadAttemptCount= */ 7, + /* newFilesReceivedTimestamp= */ testClock.currentTimeMillis() - 500L); fileGroup1 = fileGroup1.toBuilder() .setDownloadConditions(DownloadConditions.getDefaultInstance()) @@ -4460,18 +4881,8 @@ public class FileGroupManagerTest { GroupKey expectedKey2 = testKey2.toBuilder().setDownloaded(false).build(); // The file status isn't changed to DOWNLOAD_COMPLETE, it remains DOWNLOAD_IN_PROGRESS. // An UNKNOWN_ERROR is logged. - when(mockDownloader.startDownloading( - eq(expectedKey2), - anyInt(), - anyLong(), - any(Uri.class), - any(String.class), - anyInt(), - any(DownloadConditions.class), - isA(DownloaderCallbackImpl.class), - anyInt(), - anyList())) - .thenReturn(Futures.immediateVoidFuture()); + when(mockDownloader.getInProgressFuture(any(String.class), any(Uri.class))) + .thenReturn(Futures.immediateFuture(Optional.of(Futures.immediateVoidFuture()))); DataFileGroupInternal tmpFileGroup3 = MddTestUtil.createDataFileGroupInternal(TEST_GROUP_3, 2); final DataFileGroupInternal fileGroup3 = @@ -4484,15 +4895,17 @@ public class FileGroupManagerTest { writeSharedFiles( sharedFilesMetadata, fileGroup3, - ImmutableList.of(FileStatus.DOWNLOAD_COMPLETE, FileStatus.DOWNLOAD_IN_PROGRESS)); + ImmutableList.of(FileStatus.DOWNLOAD_COMPLETE, FileStatus.DOWNLOAD_FAILED)); GroupKey expectedKey3 = testKey3.toBuilder().setDownloaded(false).build(); - // One file fails, new status is DOWNLOAD_FAILED but the downloader returns an + // One file fails, status is DOWNLOAD_FAILED but the downloader returns an // immediateVoidFuture. An UNKNOWN_ERROR is logged. when(mockDownloader.startDownloading( + any(String.class), eq(expectedKey3), anyInt(), anyLong(), + any(String.class), any(Uri.class), any(String.class), anyInt(), @@ -4513,6 +4926,53 @@ public class FileGroupManagerTest { }); fileGroupManager.scheduleAllPendingGroupsForDownload(true, noCustomValidation()).get(); + + verify(mockLogger) + .logMddDownloadLatency( + createFileGroupDetails(fileGroup1).build(), + createMddDownloadLatency( + /* downloadAttemptCount= */ 8, + /* downloadLatencyMs= */ 0L, + /* totalLatencyMs= */ 500L)); + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP_2, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP_3, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); + + // Make sure that the successful download of fileGroup1, the failed downloads of fileGroup2 and + // fileGroup3 are all logged to clearcut. + verify(mockLogger) + .logMddDownloadResult( + MddDownloadResult.Code.SUCCESS, + createFileGroupDetails(fileGroup1) + .setOwnerPackage(context.getPackageName()) + .clearFileCount() + .build()); + verify(mockLogger) + .logMddDownloadResult( + MddDownloadResult.Code.UNKNOWN_ERROR, + createFileGroupDetails(fileGroup2) + .setOwnerPackage(context.getPackageName()) + .clearFileCount() + .build()); + + verify(mockLogger) + .logMddDownloadResult( + MddDownloadResult.Code.UNKNOWN_ERROR, + createFileGroupDetails(fileGroup3) + .setOwnerPackage(context.getPackageName()) + .clearFileCount() + .build()); } @Test @@ -4544,18 +5004,8 @@ public class FileGroupManagerTest { fileGroupManager.scheduleAllPendingGroupsForDownload(false, noCustomValidation()).get(); // Only the files in the first group will be downloaded. - verify(mockDownloader, times(2)) - .startDownloading( - eq(getPendingKey(testKey)), - anyInt(), - anyLong(), - any(Uri.class), - any(String.class), - anyInt(), - any(DownloadConditions.class), - isA(DownloaderCallbackImpl.class), - anyInt(), - anyList()); + verify(mockDownloader, times(2)).getInProgressFuture(any(String.class), any(Uri.class)); + verifyNoMoreInteractions(mockDownloader); } @@ -4590,9 +5040,11 @@ public class FileGroupManagerTest { ArgumentCaptor<DownloadConditions> downloadConditionCaptor = ArgumentCaptor.forClass(DownloadConditions.class); when(mockDownloader.startDownloading( + any(String.class), any(GroupKey.class), anyInt(), anyLong(), + any(String.class), any(Uri.class), any(String.class), anyInt(), @@ -4622,7 +5074,7 @@ public class FileGroupManagerTest { writeSharedFiles( sharedFilesMetadata, fileGroup1, - ImmutableList.of(FileStatus.DOWNLOAD_COMPLETE, FileStatus.DOWNLOAD_IN_PROGRESS)); + ImmutableList.of(FileStatus.DOWNLOAD_COMPLETE, FileStatus.SUBSCRIBED)); NewFileKey[] keys1 = MddTestUtil.createFileKeysForDataFileGroupInternal(fileGroup1); DataFileGroupInternal fileGroup2 = MddTestUtil.createDataFileGroupInternal(TEST_GROUP_2, 1); @@ -4643,6 +5095,37 @@ public class FileGroupManagerTest { fileDownloadFails(keys1[1], failingFileUri, DownloadResultCode.LOW_DISK_ERROR); fileGroupManager.scheduleAllPendingGroupsForDownload(true, noCustomValidation()).get(); + + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP_2, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); + + verify(mockLogger) + .logMddDownloadResult( + MddDownloadResult.Code.LOW_DISK_ERROR, + createFileGroupDetails(fileGroup1) + .clearFileCount() + .setOwnerPackage(context.getPackageName()) + .build()); + + verify(mockLogger) + .logMddDownloadResult( + MddDownloadResult.Code.SUCCESS, + createFileGroupDetails(fileGroup2) + .clearFileCount() + .setOwnerPackage(context.getPackageName()) + .build()); } // case 1: the file is already shared in the blob storage. @@ -4676,6 +5159,14 @@ public class FileGroupManagerTest { assertThat(sharedFileManager.getSharedFile(newFileKey).get()) .isEqualTo(existingDownloadedSharedFile); + + ArgumentCaptor<Void> mddAndroidSharingLogArgumentCaptor = ArgumentCaptor.forClass(Void.class); + + verify(mockLogger).logMddAndroidSharingLog(mddAndroidSharingLogArgumentCaptor.capture()); + Void mddAndroidSharingLog = null; + assertThat(mddAndroidSharingLogArgumentCaptor.getAllValues()) + .containsExactly(mddAndroidSharingLog); + verifyNoMoreInteractions(mockLogger); } // case 2a: the to-be-shared file is available in the blob storage. @@ -4723,6 +5214,12 @@ public class FileGroupManagerTest { assertThat(sharedFileManager.getOnDeviceUri(newFileKey).get()).isEqualTo(blobUri); ArgumentCaptor<Void> mddAndroidSharingLogArgumentCaptor = ArgumentCaptor.forClass(Void.class); + + verify(mockLogger).logMddAndroidSharingLog(mddAndroidSharingLogArgumentCaptor.capture()); + Void mddAndroidSharingLog = null; + assertThat(mddAndroidSharingLogArgumentCaptor.getAllValues()) + .containsExactly(mddAndroidSharingLog); + verifyNoMoreInteractions(mockLogger); } // case 3: the to-be-shared file is available in the local storage. @@ -4768,7 +5265,7 @@ public class FileGroupManagerTest { newFileKey.getChecksum(), mockSilentFeedback, /* instanceId= */ Optional.absent(), - /* androidShared = */ false); + /* androidShared= */ false); assertThat(fileStorage.exists(onDeviceuri)).isTrue(); fileGroupManager.tryToShareBeforeDownload(fileGroup, file, newFileKey).get(); @@ -4789,6 +5286,13 @@ public class FileGroupManagerTest { onDeviceFile.delete(); ArgumentCaptor<Void> mddAndroidSharingLogArgumentCaptor = ArgumentCaptor.forClass(Void.class); + + verify(mockLogger).logMddAndroidSharingLog(mddAndroidSharingLogArgumentCaptor.capture()); + + Void mddAndroidSharingLog = null; + assertThat(mddAndroidSharingLogArgumentCaptor.getAllValues()) + .containsExactly(mddAndroidSharingLog); + verifyNoMoreInteractions(mockLogger); } // The file can't be shared and isn't available locally. @@ -4823,6 +5327,8 @@ public class FileGroupManagerTest { SharedFile sharedFile = sharedFileManager.getSharedFile(newFileKey).get(); assertThat(sharedFile).isEqualTo(existingSharedFile); + + verifyNoInteractions(mockLogger); } // case 4: the non-to-be-shared file can't be shared and is available in the local storage. @@ -4858,6 +5364,8 @@ public class FileGroupManagerTest { verify(mockSharedFileManager, never()) .setAndroidSharedDownloadedFileEntry(any(), any(), anyLong()); + + verifyNoInteractions(mockLogger); } @Test @@ -4906,6 +5414,14 @@ public class FileGroupManagerTest { SharedFile sharedFile = sharedFileManager.getSharedFile(newFileKey).get(); assertThat(sharedFile).isEqualTo(existingSharedFile); + + ArgumentCaptor<Void> mddAndroidSharingLogArgumentCaptor = ArgumentCaptor.forClass(Void.class); + verify(mockLogger).logMddAndroidSharingLog(mddAndroidSharingLogArgumentCaptor.capture()); + + Void mddAndroidSharingLog = null; + assertThat(mddAndroidSharingLogArgumentCaptor.getAllValues()) + .containsExactly(mddAndroidSharingLog); + verifyNoMoreInteractions(mockLogger); } @Test @@ -4950,6 +5466,14 @@ public class FileGroupManagerTest { // openForWrite is called only once for acquiring the lease. verify(mockBackend, never()).openForWrite(blobUri); verify(mockBackend).openForWrite(leaseUri); + + ArgumentCaptor<Void> mddAndroidSharingLogArgumentCaptor = ArgumentCaptor.forClass(Void.class); + + verify(mockLogger).logMddAndroidSharingLog(mddAndroidSharingLogArgumentCaptor.capture()); + Void mddAndroidSharingLog = null; + assertThat(mddAndroidSharingLogArgumentCaptor.getAllValues()) + .containsExactly(mddAndroidSharingLog); + verifyNoMoreInteractions(mockLogger); } @Test @@ -4987,6 +5511,13 @@ public class FileGroupManagerTest { assertThat(sharedFile).isEqualTo(existingSharedFile); ArgumentCaptor<Void> mddAndroidSharingLogArgumentCaptor = ArgumentCaptor.forClass(Void.class); + + verify(mockLogger).logMddAndroidSharingLog(mddAndroidSharingLogArgumentCaptor.capture()); + + Void mddAndroidSharingLog = null; + assertThat(mddAndroidSharingLogArgumentCaptor.getAllValues()) + .containsExactly(mddAndroidSharingLog); + verifyNoMoreInteractions(mockLogger); } @Test @@ -5030,6 +5561,14 @@ public class FileGroupManagerTest { SharedFile sharedFile = sharedFileManager.getSharedFile(newFileKey).get(); // Since there was an exception, the existing shared file didn't update the expiration date. assertThat(sharedFile).isEqualTo(existingSharedFile); + + ArgumentCaptor<Void> mddAndroidSharingLogArgumentCaptor = ArgumentCaptor.forClass(Void.class); + verify(mockLogger).logMddAndroidSharingLog(mddAndroidSharingLogArgumentCaptor.capture()); + + Void mddAndroidSharingLog = null; + assertThat(mddAndroidSharingLogArgumentCaptor.getAllValues()) + .containsExactly(mddAndroidSharingLog); + verifyNoMoreInteractions(mockLogger); } @Test @@ -5059,6 +5598,8 @@ public class FileGroupManagerTest { SharedFile sharedFile = sharedFileManager.getSharedFile(newFileKey).get(); assertThat(sharedFile).isEqualTo(existingSharedFile); + + verifyNoInteractions(mockLogger); } @Test @@ -5100,6 +5641,14 @@ public class FileGroupManagerTest { assertThat(sharedFile.getMaxExpirationDateSecs()).isEqualTo(FILE_GROUP_EXPIRATION_DATE_SECS); assertThat(sharedFile.getAndroidShared()).isTrue(); assertThat(sharedFileManager.getOnDeviceUri(newFileKey).get()).isEqualTo(blobUri); + + ArgumentCaptor<Void> mddAndroidSharingLogArgumentCaptor = ArgumentCaptor.forClass(Void.class); + + verify(mockLogger).logMddAndroidSharingLog(mddAndroidSharingLogArgumentCaptor.capture()); + Void mddAndroidSharingLog = null; + assertThat(mddAndroidSharingLogArgumentCaptor.getAllValues()) + .containsExactly(mddAndroidSharingLog); + verifyNoMoreInteractions(mockLogger); } @Test @@ -5153,8 +5702,16 @@ public class FileGroupManagerTest { assertThat(sharedFile.getAndroidShared()).isTrue(); assertThat(sharedFileManager.getOnDeviceUri(newFileKey).get()).isEqualTo(blobUri); - // Local copy has been deleted. - assertThat(fileStorage.exists(onDeviceuri)).isFalse(); + // Local copy has not been deleted. + assertThat(fileStorage.exists(onDeviceuri)).isTrue(); + + ArgumentCaptor<Void> mddAndroidSharingLogArgumentCaptor = ArgumentCaptor.forClass(Void.class); + + verify(mockLogger).logMddAndroidSharingLog(mddAndroidSharingLogArgumentCaptor.capture()); + Void mddAndroidSharingLog = null; + assertThat(mddAndroidSharingLogArgumentCaptor.getAllValues()) + .containsExactly(mddAndroidSharingLog); + verifyNoMoreInteractions(mockLogger); } @Test @@ -5211,8 +5768,16 @@ public class FileGroupManagerTest { assertThat(sharedFile.getAndroidShared()).isTrue(); assertThat(sharedFileManager.getOnDeviceUri(newFileKey).get()).isEqualTo(blobUri); - // Local copy has been deleted. - assertThat(fileStorage.exists(onDeviceuri)).isFalse(); + // Local copy has not been deleted. + assertThat(fileStorage.exists(onDeviceuri)).isTrue(); + + ArgumentCaptor<Void> mddAndroidSharingLogArgumentCaptor = ArgumentCaptor.forClass(Void.class); + + verify(mockLogger).logMddAndroidSharingLog(mddAndroidSharingLogArgumentCaptor.capture()); + Void mddAndroidSharingLog = null; + assertThat(mddAndroidSharingLogArgumentCaptor.getAllValues()) + .containsExactly(mddAndroidSharingLog); + verifyNoMoreInteractions(mockLogger); } @Test @@ -5264,6 +5829,8 @@ public class FileGroupManagerTest { // Local copy still available. assertThat(fileStorage.exists(onDeviceuri)).isTrue(); onDeviceFile.delete(); + + verifyNoInteractions(mockLogger); } @Test @@ -5318,6 +5885,15 @@ public class FileGroupManagerTest { // Local copy still available. assertThat(fileStorage.exists(onDeviceuri)).isTrue(); onDeviceFile.delete(); + + ArgumentCaptor<Void> mddAndroidSharingLogArgumentCaptor = ArgumentCaptor.forClass(Void.class); + verify(mockLogger).logMddAndroidSharingLog(mddAndroidSharingLogArgumentCaptor.capture()); + + Void mddAndroidSharingLog = null; + assertThat(mddAndroidSharingLogArgumentCaptor.getAllValues()) + .containsExactly(mddAndroidSharingLog); + + verifyNoMoreInteractions(mockLogger); } @Test @@ -5365,6 +5941,14 @@ public class FileGroupManagerTest { SharedFile sharedFile = sharedFileManager.getSharedFile(newFileKey).get(); assertThat(sharedFile).isEqualTo(existingSharedFile); + + ArgumentCaptor<Void> mddAndroidSharingLogArgumentCaptor = ArgumentCaptor.forClass(Void.class); + verify(mockLogger).logMddAndroidSharingLog(mddAndroidSharingLogArgumentCaptor.capture()); + + Void mddAndroidSharingLog = null; + assertThat(mddAndroidSharingLogArgumentCaptor.getAllValues()) + .containsExactly(mddAndroidSharingLog); + verifyNoMoreInteractions(mockLogger); } @Test @@ -5384,6 +5968,14 @@ public class FileGroupManagerTest { ExecutionException exception = assertThrows(ExecutionException.class, tryToShareFuture::get); assertThat(exception).hasCauseThat().isInstanceOf(SharedFileMissingException.class); + ArgumentCaptor<Void> mddAndroidSharingLogArgumentCaptor = ArgumentCaptor.forClass(Void.class); + verify(mockLogger).logMddAndroidSharingLog(mddAndroidSharingLogArgumentCaptor.capture()); + + Void mddAndroidSharingLog = null; + assertThat(mddAndroidSharingLogArgumentCaptor.getAllValues()) + .containsExactly(mddAndroidSharingLog); + verifyNoMoreInteractions(mockLogger); + verify(mockBackend, never()).exists(any()); verify(mockBackend, never()).openForWrite(any()); verify(mockSharedFileManager, never()) @@ -5441,6 +6033,13 @@ public class FileGroupManagerTest { .updateMaxExpirationDateSecs(newFileKey, FILE_GROUP_EXPIRATION_DATE_SECS); assertThat(fileStorage.exists(onDeviceuri)).isTrue(); onDeviceFile.delete(); + + ArgumentCaptor<Void> mddAndroidSharingLogArgumentCaptor = ArgumentCaptor.forClass(Void.class); + verify(mockLogger).logMddAndroidSharingLog(mddAndroidSharingLogArgumentCaptor.capture()); + Void mddAndroidSharingLog = null; + assertThat(mddAndroidSharingLogArgumentCaptor.getAllValues()) + .containsExactly(mddAndroidSharingLog); + verifyNoMoreInteractions(mockLogger); } @Test @@ -5503,6 +6102,14 @@ public class FileGroupManagerTest { verify(mockSharedFileManager).updateMaxExpirationDateSecs(newFileKey, 0); assertThat(fileStorage.exists(onDeviceuri)).isTrue(); onDeviceFile.delete(); + + ArgumentCaptor<Void> mddAndroidSharingLogArgumentCaptor = ArgumentCaptor.forClass(Void.class); + verify(mockLogger).logMddAndroidSharingLog(mddAndroidSharingLogArgumentCaptor.capture()); + + Void mddAndroidSharingLog = null; + assertThat(mddAndroidSharingLogArgumentCaptor.getAllValues()) + .containsExactly(mddAndroidSharingLog); + verifyNoMoreInteractions(mockLogger); } @Test @@ -5557,6 +6164,14 @@ public class FileGroupManagerTest { // Local copy still available. assertThat(fileStorage.exists(onDeviceuri)).isTrue(); onDeviceFile.delete(); + + ArgumentCaptor<Void> mddAndroidSharingLogArgumentCaptor = ArgumentCaptor.forClass(Void.class); + verify(mockLogger).logMddAndroidSharingLog(mddAndroidSharingLogArgumentCaptor.capture()); + + Void mddAndroidSharingLog = null; + assertThat(mddAndroidSharingLogArgumentCaptor.getAllValues()) + .containsExactly(mddAndroidSharingLog); + verifyNoMoreInteractions(mockLogger); } @Test @@ -5622,47 +6237,14 @@ public class FileGroupManagerTest { // Local copy still available. assertThat(fileStorage.exists(onDeviceuri)).isTrue(); onDeviceFile.delete(); - } - - @Test - public void tryToShareAfterDownload_blobExists_deleteLocalCopyFails() throws Exception { - // Create a file group with expiration date bigger than the expiration date of the existing - // SharedFile. - DataFileGroupInternal fileGroup = - MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 1).toBuilder() - .setExpirationDateSecs(FILE_GROUP_EXPIRATION_DATE_SECS) - .setDownloadConditions(DownloadConditions.getDefaultInstance()) - .build(); - - DataFile file = MddTestUtil.createSharedDataFile("fileId", 0); - NewFileKey newFileKey = - SharedFilesMetadata.createKeyFromDataFile(file, AllowedReaders.ALL_GOOGLE_APPS); - SharedFile existingSharedFile = - SharedFile.newBuilder() - .setFileStatus(FileStatus.DOWNLOAD_COMPLETE) - .setFileName("fileName") - .setAndroidShared(false) - .build(); - sharedFilesMetadata.write(newFileKey, existingSharedFile).get(); - - Uri blobUri = DirectoryUtil.getBlobUri(context, file.getAndroidSharingChecksum()); - Uri leaseUri = - DirectoryUtil.getBlobStoreLeaseUri( - context, file.getAndroidSharingChecksum(), FILE_GROUP_EXPIRATION_DATE_SECS); - // The file is available in the blob storage - when(mockBackend.exists(blobUri)).thenReturn(true); - fileGroupManager.tryToShareAfterDownload(fileGroup, file, newFileKey).get(); - - // openForWrite is called only once for acquiring the lease. - verify(mockBackend).exists(blobUri); - verify(mockBackend).openForWrite(leaseUri); + ArgumentCaptor<Void> mddAndroidSharingLogArgumentCaptor = ArgumentCaptor.forClass(Void.class); + verify(mockLogger).logMddAndroidSharingLog(mddAndroidSharingLogArgumentCaptor.capture()); - SharedFile sharedFile = sharedFileManager.getSharedFile(newFileKey).get(); - // Verify that the SharedFile has updated its expiration date after the download. - assertThat(sharedFile.getMaxExpirationDateSecs()).isEqualTo(FILE_GROUP_EXPIRATION_DATE_SECS); - assertThat(sharedFile.getAndroidShared()).isTrue(); - assertThat(sharedFileManager.getOnDeviceUri(newFileKey).get()).isEqualTo(blobUri); + Void mddAndroidSharingLog = null; + assertThat(mddAndroidSharingLogArgumentCaptor.getAllValues()) + .containsExactly(mddAndroidSharingLog); + verifyNoMoreInteractions(mockLogger); } @Test @@ -5687,12 +6269,22 @@ public class FileGroupManagerTest { assertThat( fileGroupManager - .verifyPendingGroupDownloaded(testKey, fileGroup1, noCustomValidation()) + .verifyGroupDownloaded( + testKey, + fileGroup1, + /* removePendingVersion= */ true, + noCustomValidation(), + DownloadStateLogger.forDownload(mockLogger)) .get()) .isEqualTo(GroupDownloadStatus.PENDING); assertThat( fileGroupManager - .verifyPendingGroupDownloaded(testKey2, fileGroup2, noCustomValidation()) + .verifyGroupDownloaded( + testKey2, + fileGroup2, + /* removePendingVersion= */ true, + noCustomValidation(), + DownloadStateLogger.forDownload(mockLogger)) .get()) .isEqualTo(GroupDownloadStatus.DOWNLOADED); @@ -5708,6 +6300,21 @@ public class FileGroupManagerTest { // Verify that the completely downloaded group is written into metadata. DataFileGroupInternal downloadedGroup2 = readDownloadedFileGroup(testKey2); assertThat(downloadedGroup2).isEqualTo(fileGroup2); + + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP_2, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); } @Test @@ -5743,6 +6350,21 @@ public class FileGroupManagerTest { // Verify that the completely downloaded group is written into metadata. DataFileGroupInternal downloadedGroup2 = readDownloadedFileGroup(testKey2); assertThat(downloadedGroup2).isEqualTo(fileGroup2); + + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP_2, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); } @Test @@ -5797,6 +6419,21 @@ public class FileGroupManagerTest { DataFileGroupInternal downloadedGroup4 = readDownloadedFileGroup(testKey3); assertThat(downloadedGroup4).isEqualTo(fileGroup4); + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP_2, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); + // fileGroup3 should have been scheduled for deletion. fileGroup3 = fileGroup3.toBuilder() @@ -5833,6 +6470,21 @@ public class FileGroupManagerTest { fileGroup2 = FileGroupUtil.setDownloadedTimestampInMillis(fileGroup2, 1000); DataFileGroupInternal downloadedGroup2 = readDownloadedFileGroup(testKey2); assertThat(downloadedGroup2).isEqualTo(fileGroup2); + + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP_2, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); } @Test @@ -5917,6 +6569,8 @@ public class FileGroupManagerTest { assertThat(fileGroupsMetadata.getAllGroupKeys().get()) .containsExactly(getDownloadedKey(key1), getDownloadedKey(key2), getDownloadedKey(key3)); + + verifyNoInteractions(mockLogger); } @Test @@ -5957,6 +6611,15 @@ public class FileGroupManagerTest { assertThat(fileGroupsMetadata.getAllGroupKeys().get()) .containsExactly(getDownloadedKey(key1), getDownloadedKey(key3)); + + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP_2, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); + verifyNoMoreInteractions(mockLogger); } @Test @@ -6013,6 +6676,22 @@ public class FileGroupManagerTest { pendingGroupKeyWithFileMissing, groupKeyWithNoFileMissing); } + + verify(mockLogger, times(2)) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); + verify(mockLogger) + .logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP_2, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); + verifyNoMoreInteractions(mockLogger); } @Test @@ -6085,6 +6764,117 @@ public class FileGroupManagerTest { .isEqualTo("android"); } + @Test + public void testAddGroupForDownload_withExperimentationConfig() throws Exception { + flags.enableDownloadStageExperimentIdPropagation = Optional.of(true); + + Long buildId = 999L; + Integer experimentId = 12345; + DataFileGroupInternal dataFileGroup = + MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 2).toBuilder() + .setBuildId(buildId) + .build(); + + assertThat(fileGroupManager.addGroupForDownload(testKey, dataFileGroup).get()).isTrue(); + } + + @Test + public void testAddGroupForDownload_withExperimentationConfig_overwritesPendingExperimentIds() + throws Exception { + flags.enableDownloadStageExperimentIdPropagation = Optional.of(true); + + long buildId = 999L; + long buildId2 = 100L; + int experimentId = 12345; + int experimentId2 = 23456; + + DataFileGroupInternal dataFileGroup = + MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 2).toBuilder() + .setBuildId(buildId) + .build(); + + DataFileGroupInternal dataFileGroup2 = + MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 2).toBuilder() + .setBuildId(buildId2) + .build(); + + assertThat(fileGroupManager.addGroupForDownload(testKey, dataFileGroup).get()).isTrue(); + // Overwrite the group. The old experiment id should be deleted and the new experiment id should + // be populated. + assertThat(fileGroupManager.addGroupForDownload(testKey, dataFileGroup2).get()).isTrue(); + } + + @Test + public void testDownloadPendingGroup_withExperimentationConfig_updatesExperimentIdToDownloaded() + throws Exception { + flags.enableDownloadStageExperimentIdPropagation = Optional.of(true); + + int experimentIdDownloading = 12345; + int experimentIdDownloaded = 23456; + long buildId = 999L; + + ExtraHttpHeader extraHttpHeader = + ExtraHttpHeader.newBuilder().setKey("user-agent").setValue("mdd-downloader").build(); + + // Write 1 group to the pending shared prefs. + DataFileGroupInternal fileGroup = + createDataFileGroup( + TEST_GROUP, + /* fileCount= */ 2, + /* downloadAttemptCount= */ 3, + /* newFilesReceivedTimestamp= */ testClock.currentTimeMillis() - 500L) + .toBuilder() + .setBuildId(buildId) + .setOwnerPackage(context.getPackageName()) + .setDownloadConditions(DownloadConditions.getDefaultInstance()) + .setTrafficTag(TRAFFIC_TAG) + .addGroupExtraHttpHeaders(extraHttpHeader) + .build(); + + writePendingFileGroup(testKey, fileGroup); + + writeSharedFiles( + sharedFilesMetadata, + fileGroup, + ImmutableList.of(FileStatus.DOWNLOAD_COMPLETE, FileStatus.DOWNLOAD_COMPLETE)); + + fileGroupManager + .downloadFileGroup(testKey, DownloadConditions.getDefaultInstance(), noCustomValidation()) + .get(); + } + + @Test + public void testRemoveFileGroup_withExperimentationConfig_removesExperimentIds() + throws Exception { + flags.enableDownloadStageExperimentIdPropagation = Optional.of(true); + + long buildId = 999L; + int experimentId = 12345; + DataFileGroupInternal dataFileGroup = + MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 2).toBuilder() + .setBuildId(buildId) + .build(); + + assertThat(fileGroupManager.addGroupForDownload(testKey, dataFileGroup).get()).isTrue(); + fileGroupManager.removeFileGroup(testKey, /* pendingOnly= */ false).get(); + } + + @Test + public void testRemoveFileGroups_withExperimentationConfig_removesExperimentIds() + throws Exception { + flags.enableDownloadStageExperimentIdPropagation = Optional.of(true); + + long buildId = 999L; + int experimentId = 12345; + DataFileGroupInternal dataFileGroup = + MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 2).toBuilder() + .setBuildId(buildId) + .build(); + + assertThat(fileGroupManager.addGroupForDownload(testKey, dataFileGroup).get()).isTrue(); + fileGroupManager.removeFileGroups(ImmutableList.of(testKey)).get(); + } + /** * Re-instantiates {@code fileGroupManager} with the injected parameters. * @@ -6092,10 +6882,18 @@ public class FileGroupManagerTest { */ private void resetFileGroupManager( FileGroupsMetadata fileGroupsMetadata, SharedFileManager sharedFileManager) throws Exception { + resetFileGroupManager(this.mockLogger, fileGroupsMetadata, sharedFileManager); + } + + private void resetFileGroupManager( + EventLogger eventLogger, + FileGroupsMetadata fileGroupsMetadata, + SharedFileManager sharedFileManager) + throws Exception { fileGroupManager = new FileGroupManager( context, - mockLogger, + eventLogger, mockSilentFeedback, fileGroupsMetadata, sharedFileManager, @@ -6108,8 +6906,15 @@ public class FileGroupManagerTest { flags); } - private static Void createFileGroupDetails(DataFileGroupInternal fileGroup) { - return null; + private static DataDownloadFileGroupStats.Builder createFileGroupDetails( + DataFileGroupInternal fileGroup) { + return DataDownloadFileGroupStats.newBuilder() + .setOwnerPackage(fileGroup.getOwnerPackage()) + .setFileGroupName(fileGroup.getGroupName()) + .setFileGroupVersionNumber(fileGroup.getFileGroupVersionNumber()) + .setBuildId(fileGroup.getBuildId()) + .setVariantId(fileGroup.getVariantId()) + .setFileCount(fileGroup.getFileCount()); } private static Void createMddDownloadLatency( @@ -6130,9 +6935,11 @@ public class FileGroupManagerTest { /** The file download succeeds so the new file status is DOWNLOAD_COMPLETE. */ private void fileDownloadSucceeds(NewFileKey key, Uri fileUri) { when(mockDownloader.startDownloading( + any(String.class), any(GroupKey.class), anyInt(), anyLong(), + any(String.class), eq(fileUri), any(String.class), anyInt(), @@ -6160,9 +6967,11 @@ public class FileGroupManagerTest { */ private void fileDownloadFails(NewFileKey key, Uri fileUri, DownloadResultCode failureCode) { when(mockDownloader.startDownloading( + any(String.class), any(GroupKey.class), anyInt(), anyLong(), + any(String.class), eq(fileUri), any(String.class), anyInt(), @@ -6262,8 +7071,8 @@ public class FileGroupManagerTest { dataFile.getFileId(), newFileKey.getChecksum(), mockSilentFeedback, - /* instanceId = */ Optional.absent(), - /* androidShared = */ false)); + /* instanceId= */ Optional.absent(), + /* androidShared= */ false)); } return uriList; } diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/FileGroupsMetadataTest.java b/javatests/com/google/android/libraries/mobiledatadownload/internal/FileGroupsMetadataTest.java index 0fe4f28..abe9571 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/internal/FileGroupsMetadataTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/FileGroupsMetadataTest.java @@ -20,11 +20,16 @@ import static com.google.common.truth.Truth.assertWithMessage; import android.content.Context; import android.content.SharedPreferences; -import android.util.Pair; +import android.net.Uri; import androidx.test.core.app.ApplicationProvider; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; +import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; +import com.google.mobiledatadownload.internal.MetadataProto.GroupKeyProperties; import com.google.android.libraries.mobiledatadownload.SilentFeedback; import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; import com.google.android.libraries.mobiledatadownload.file.backends.AndroidFileBackend; +import com.google.android.libraries.mobiledatadownload.file.backends.AndroidUri; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup; import com.google.android.libraries.mobiledatadownload.internal.logging.EventLogger; import com.google.android.libraries.mobiledatadownload.internal.util.DirectoryUtil; import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupUtil; @@ -37,9 +42,6 @@ import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.MoreExecutors; import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; -import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; -import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; -import com.google.mobiledatadownload.internal.MetadataProto.GroupKeyProperties; import java.io.File; import java.time.Duration; import java.util.ArrayList; @@ -98,7 +100,10 @@ public class FileGroupsMetadataTest { private Context context; private FakeTimeSource testClock; private FileGroupsMetadata fileGroupsMetadata; + private Uri destinationUri; + private Uri diagnosticUri; private final TestFlags flags = new TestFlags(); + @Mock EventLogger mockLogger; @Mock SilentFeedback mockSilentFeedback; @@ -134,6 +139,16 @@ public class FileGroupsMetadataTest { new SynchronousFileStorage(Arrays.asList(AndroidFileBackend.builder(context).build())); testClock = new FakeTimeSource(); + destinationUri = + AndroidUri.builder(context) + .setPackage(context.getPackageName()) + .setRelativePath("dest.pb") + .build(); + diagnosticUri = + AndroidUri.builder(context) + .setPackage(context.getPackageName()) + .setRelativePath("diag.pb") + .build(); SharedPreferencesFileGroupsMetadata sharedPreferencesImpl = new SharedPreferencesFileGroupsMetadata( context, testClock, mockSilentFeedback, instanceId, CONTROL_EXECUTOR); @@ -146,12 +161,18 @@ public class FileGroupsMetadataTest { @After public void tearDown() throws Exception { + if (fileStorage.exists(diagnosticUri)) { + fileStorage.deleteFile(diagnosticUri); + } + if (fileStorage.exists(destinationUri)) { + fileStorage.deleteFile(destinationUri); + } fileGroupsMetadata.clear().get(); } @Test public void serializeAndDeserializeFileGroupKey() throws Exception { - String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(testKey, context); + String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(testKey); GroupKey deserializedGroupKey = FileGroupsMetadataUtil.deserializeGroupKey(serializedGroupKey); assertThat(deserializedGroupKey.getGroupName()).isEqualTo(TEST_GROUP); @@ -326,8 +347,7 @@ public class FileGroupsMetadataTest { prefs.edit().putString("garbage-key", "garbage-value").commit(); } - List<Pair<GroupKey, DataFileGroupInternal>> allGroups = - fileGroupsMetadata.getAllFreshGroups().get(); + List<GroupKeyAndGroup> allGroups = fileGroupsMetadata.getAllFreshGroups().get(); assertThat(allGroups).hasSize(3); verifyNoErrorInPdsMigration(); @@ -590,7 +610,7 @@ public class FileGroupsMetadataTest { */ boolean writeDataFileGroup( GroupKey groupKey, DataFileGroup fileGroup, Optional<String> instanceId) { - String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey, context); + String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey); SharedPreferences prefs = SharedPreferencesUtil.getSharedPreferences( context, FileGroupsMetadataUtil.MDD_FILE_GROUPS, instanceId); diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/MddIsolatedStructuresTest.java b/javatests/com/google/android/libraries/mobiledatadownload/internal/MddIsolatedStructuresTest.java index 582f412..1e04b29 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/internal/MddIsolatedStructuresTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/MddIsolatedStructuresTest.java @@ -18,11 +18,22 @@ package com.google.android.libraries.mobiledatadownload.internal; import static com.google.common.truth.Truth.assertThat; import static java.nio.charset.StandardCharsets.UTF_16; +import android.accounts.Account; import android.content.Context; import android.net.Uri; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.mobiledatadownload.internal.MetadataProto.DataFile; +import com.google.mobiledatadownload.internal.MetadataProto.DataFile.ChecksumType; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal.AllowedReaders; +import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions; +import com.google.mobiledatadownload.internal.MetadataProto.FileStatus; +import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; +import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey; +import com.google.mobiledatadownload.internal.MetadataProto.SharedFile; import com.google.android.libraries.mobiledatadownload.SilentFeedback; +import com.google.android.libraries.mobiledatadownload.account.AccountUtil; import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; import com.google.android.libraries.mobiledatadownload.file.backends.AndroidFileBackend; import com.google.android.libraries.mobiledatadownload.file.backends.AndroidUri; @@ -32,22 +43,21 @@ import com.google.android.libraries.mobiledatadownload.file.openers.WriteByteArr import com.google.android.libraries.mobiledatadownload.internal.downloader.MddFileDownloader; import com.google.android.libraries.mobiledatadownload.internal.experimentation.NoOpDownloadStageManager; import com.google.android.libraries.mobiledatadownload.internal.logging.EventLogger; +import com.google.android.libraries.mobiledatadownload.internal.logging.LoggingStateStore; import com.google.android.libraries.mobiledatadownload.internal.util.SymlinkUtil; +import com.google.android.libraries.mobiledatadownload.monitor.NetworkUsageMonitor; +import com.google.android.libraries.mobiledatadownload.testing.BlockingFileDownloader; import com.google.android.libraries.mobiledatadownload.testing.FakeTimeSource; +import com.google.android.libraries.mobiledatadownload.testing.MddTestDependencies; import com.google.android.libraries.mobiledatadownload.testing.TestFlags; import com.google.common.base.Optional; import com.google.common.util.concurrent.AsyncFunction; import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; -import com.google.mobiledatadownload.internal.MetadataProto.DataFile; -import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; -import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal.AllowedReaders; -import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions; -import com.google.mobiledatadownload.internal.MetadataProto.FileStatus; -import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; -import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey; -import com.google.mobiledatadownload.internal.MetadataProto.SharedFile; import java.util.Arrays; +import java.util.Random; import java.util.concurrent.Executor; import java.util.concurrent.Executors; import org.junit.Before; @@ -67,6 +77,11 @@ public final class MddIsolatedStructuresTest { private static final String TEST_GROUP = "test-group"; + private static final String TEST_ACCOUNT_1 = + AccountUtil.serialize(new Account("com.google", "test1")); + private static final String TEST_ACCOUNT_2 = + AccountUtil.serialize(new Account("com.google", "test2")); + @Rule public TemporaryUri tempUri = new TemporaryUri(); private Context context; @@ -77,21 +92,28 @@ public final class MddIsolatedStructuresTest { private FakeTimeSource testClock; private SynchronousFileStorage fileStorage; private FakeFileBackend fakeAndroidFileBackend; - @Mock SilentFeedback mockSilentFeedback; + private BlockingFileDownloader blockingFileDownloader; + private MddFileDownloader mddFileDownloader; + private LoggingStateStore loggingStateStore; GroupKey defaultGroupKey; DataFileGroupInternal defaultFileGroup; DataFile file; NewFileKey newFileKey; - SharedFile existingSharedFile; + SharedFile existingDownloadedSharedFile; - @Mock MddFileDownloader mockDownloader; + @Mock SilentFeedback mockSilentFeedback; @Mock EventLogger mockLogger; + @Mock NetworkUsageMonitor mockNetworkUsageMonitor; @Rule public final MockitoRule mockito = MockitoJUnit.rule(); private static final Executor SEQUENTIAL_CONTROL_EXECUTOR = Executors.newSingleThreadScheduledExecutor(); + // Create a download executor separate from the sequential control executor + private static final ListeningExecutorService DOWNLOAD_EXECUTOR = + MoreExecutors.listeningDecorator(Executors.newSingleThreadScheduledExecutor()); + @Before public void setUp() throws Exception { context = ApplicationProvider.getApplicationContext(); @@ -100,9 +122,30 @@ public final class MddIsolatedStructuresTest { TestFlags flags = new TestFlags(); + blockingFileDownloader = new BlockingFileDownloader(DOWNLOAD_EXECUTOR); + fakeAndroidFileBackend = new FakeFileBackend(AndroidFileBackend.builder(context).build()); fileStorage = new SynchronousFileStorage(Arrays.asList(fakeAndroidFileBackend)); + loggingStateStore = + MddTestDependencies.LoggingStateStoreImpl.SHARED_PREFERENCES.loggingStateStore( + context, + Optional.absent(), + new FakeTimeSource(), + SEQUENTIAL_CONTROL_EXECUTOR, + new Random()); + + mddFileDownloader = + new MddFileDownloader( + context, + () -> blockingFileDownloader, + fileStorage, + mockNetworkUsageMonitor, + Optional.absent(), + loggingStateStore, + SEQUENTIAL_CONTROL_EXECUTOR, + flags); + fileGroupsMetadata = new SharedPreferencesFileGroupsMetadata( context, @@ -119,7 +162,7 @@ public final class MddIsolatedStructuresTest { mockSilentFeedback, sharedFilesMetadata, fileStorage, - mockDownloader, + mddFileDownloader, Optional.absent(), Optional.absent(), mockLogger, @@ -148,15 +191,22 @@ public final class MddIsolatedStructuresTest { .setGroupName(TEST_GROUP) .setOwnerPackage(context.getPackageName()) .build(); + file = + DataFile.newBuilder() + .setChecksumType(ChecksumType.NONE) + .setUrlToDownload("https://test.file") + .setFileId("my-file") + .setRelativeFilePath("mycustom/file.txt") + .build(); defaultFileGroup = - MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 1).toBuilder() + MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 0).toBuilder() .setPreserveFilenamesAndIsolateFiles(true) + .addFile(file) .build(); - file = defaultFileGroup.getFile(0); newFileKey = SharedFilesMetadata.createKeyFromDataFile(file, AllowedReaders.ALL_GOOGLE_APPS); - existingSharedFile = + existingDownloadedSharedFile = SharedFile.newBuilder() .setFileStatus(FileStatus.DOWNLOAD_COMPLETE) .setFileName("fileName") @@ -168,7 +218,8 @@ public final class MddIsolatedStructuresTest { public void testSymlinkUtil() throws Exception { Uri targetUri = AndroidUri.builder(context).setRelativePath("targetFile").build(); // Write some data so the target file exists. - fileStorage.open(targetUri, WriteByteArrayOpener.create("some bytes".getBytes(UTF_16))); + Void unused = + fileStorage.open(targetUri, WriteByteArrayOpener.create("some bytes".getBytes(UTF_16))); Uri linkUri = AndroidUri.builder(context).setRelativePath("linkFile").build(); @@ -181,11 +232,12 @@ public final class MddIsolatedStructuresTest { @Test public void testFileGroupManager_createsIsolatedStructures() throws Exception { writePendingFileGroup(defaultGroupKey, defaultFileGroup); - sharedFilesMetadata.write(newFileKey, existingSharedFile).get(); + sharedFilesMetadata.write(newFileKey, existingDownloadedSharedFile).get(); Uri onDeviceUri = fileGroupManager.getOnDeviceUri(file, defaultFileGroup).get(); // Actually write something to disk so the symlink points to something. - fileStorage.open(onDeviceUri, WriteByteArrayOpener.create("some content".getBytes(UTF_16))); + Void unused = + fileStorage.open(onDeviceUri, WriteByteArrayOpener.create("some content".getBytes(UTF_16))); // Download the file group so MDD creates the structures fileGroupManager @@ -193,17 +245,15 @@ public final class MddIsolatedStructuresTest { defaultGroupKey, DownloadConditions.getDefaultInstance(), noCustomValidation()) .get(); - Uri isolatedFileUri = - fileGroupManager.getAndVerifyIsolatedFileUri(onDeviceUri, file, defaultFileGroup); + Uri isolatedFileUri = fileGroupManager.getIsolatedFileUris(defaultFileGroup).get(file); assertThat(SymlinkUtil.readSymlink(context, isolatedFileUri)).isEqualTo(onDeviceUri); } @Test - public void testFileGroupManager_getDownloadedFileGroup_returnsNullIfIsolatedStructuresDontExist() - throws Exception { + public void testFileGroupManager_repairsIsolatedStructuresOnMaintenance() throws Exception { writePendingFileGroup(defaultGroupKey, defaultFileGroup); - sharedFilesMetadata.write(newFileKey, existingSharedFile).get(); + sharedFilesMetadata.write(newFileKey, existingDownloadedSharedFile).get(); fileGroupManager .downloadFileGroup( @@ -211,37 +261,200 @@ public final class MddIsolatedStructuresTest { .get(); Uri onDeviceUri = fileGroupManager.getOnDeviceUri(file, defaultFileGroup).get(); - Uri isolatedFileUri = - fileGroupManager.getAndVerifyIsolatedFileUri(onDeviceUri, file, defaultFileGroup); + Uri isolatedFileUri = fileGroupManager.getIsolatedFileUris(defaultFileGroup).get(file); + + assertThat(fileGroupManager.getFileGroup(defaultGroupKey, true).get()).isNotNull(); fileStorage.deleteFile(isolatedFileUri); - assertThat(fileGroupManager.getFileGroup(defaultGroupKey, true).get()).isNull(); + fileGroupManager.verifyAndAttemptToRepairIsolatedFiles().get(); + + assertThat(fileGroupManager.getFileGroup(defaultGroupKey, true).get()).isNotNull(); + + isolatedFileUri = fileGroupManager.getIsolatedFileUris(defaultFileGroup).get(file); + + assertThat(fileStorage.exists(isolatedFileUri)).isTrue(); + assertThat(SymlinkUtil.readSymlink(context, isolatedFileUri)).isEqualTo(onDeviceUri); } @Test - public void testFileGroupManager_repairsIsolatedStructuresOnMaintenance() throws Exception { - writePendingFileGroup(defaultGroupKey, defaultFileGroup); - sharedFilesMetadata.write(newFileKey, existingSharedFile).get(); + public void testFileGroupManager_withIsolatedRoot_isolateForDifferentVariants() throws Exception { + DataFileGroupInternal fileGroupVariant1 = + defaultFileGroup.toBuilder().setVariantId("variant1").build(); + DataFileGroupInternal fileGroupVariant2 = + defaultFileGroup.toBuilder().setVariantId("variant2").build(); + + sharedFilesMetadata.write(newFileKey, existingDownloadedSharedFile).get(); + + // Get the actual uri on device (this should be the same for both variants). + Uri onDeviceUri = fileGroupManager.getOnDeviceUri(file, fileGroupVariant1).get(); + // Actually write something to disk so the symlink points to something. + Void unused = + fileStorage.open(onDeviceUri, WriteByteArrayOpener.create("some content".getBytes(UTF_16))); + + // Add the first variant and download it to create the isolated structure + fileGroupManager.addGroupForDownload(defaultGroupKey, fileGroupVariant1).get(); + fileGroupManager + .downloadFileGroup( + defaultGroupKey, DownloadConditions.getDefaultInstance(), noCustomValidation()) + .get(); + DataFileGroupInternal storedFileGroupVariant1 = + fileGroupManager.getFileGroup(defaultGroupKey, /* downloaded= */ true).get(); + + Uri isolatedFileUriVariant1 = + fileGroupManager.getIsolatedFileUris(storedFileGroupVariant1).get(file); + // Add the second variant and download it to create another isolated structure + fileGroupManager.addGroupForDownload(defaultGroupKey, fileGroupVariant2).get(); fileGroupManager .downloadFileGroup( defaultGroupKey, DownloadConditions.getDefaultInstance(), noCustomValidation()) .get(); + DataFileGroupInternal storedFileGroupVariant2 = + fileGroupManager.getFileGroup(defaultGroupKey, /* downloaded= */ true).get(); + + Uri isolatedFileUriVariant2 = + fileGroupManager.getIsolatedFileUris(storedFileGroupVariant2).get(file); + + // Check that both symlinks exist and point to the right file + assertThat(SymlinkUtil.readSymlink(context, isolatedFileUriVariant1)).isEqualTo(onDeviceUri); + assertThat(SymlinkUtil.readSymlink(context, isolatedFileUriVariant2)).isEqualTo(onDeviceUri); + + // Check that the symlinks are not equal to each other (since the roots are different); + assertThat(isolatedFileUriVariant1).isNotEqualTo(isolatedFileUriVariant2); + } + + @Test + public void testFileGroupManager_withIsolatedRoot_isolateForDifferentAccounts() throws Exception { + GroupKey account1GroupKey = defaultGroupKey.toBuilder().setAccount(TEST_ACCOUNT_1).build(); + GroupKey account2GroupKey = defaultGroupKey.toBuilder().setAccount(TEST_ACCOUNT_2).build(); + + sharedFilesMetadata.write(newFileKey, existingDownloadedSharedFile).get(); Uri onDeviceUri = fileGroupManager.getOnDeviceUri(file, defaultFileGroup).get(); - Uri isolatedFileUri = - fileGroupManager.getAndVerifyIsolatedFileUri(onDeviceUri, file, defaultFileGroup); + // Actually write something to disk so the symlink points to something. + Void unused = + fileStorage.open(onDeviceUri, WriteByteArrayOpener.create("some content".getBytes(UTF_16))); - assertThat(fileGroupManager.getFileGroup(defaultGroupKey, true).get()).isNotNull(); + // Add the first account group and download it to create the isolated structure + fileGroupManager.addGroupForDownload(account1GroupKey, defaultFileGroup).get(); + fileGroupManager + .downloadFileGroup( + account1GroupKey, DownloadConditions.getDefaultInstance(), noCustomValidation()) + .get(); + DataFileGroupInternal storedFileGroupAccount1 = + fileGroupManager.getFileGroup(account1GroupKey, /* downloaded= */ true).get(); - fileStorage.deleteFile(isolatedFileUri); + Uri isolatedFileUriAccount1 = + fileGroupManager.getIsolatedFileUris(storedFileGroupAccount1).get(file); - assertThat(fileGroupManager.getFileGroup(defaultGroupKey, true).get()).isNull(); + // Add the second account group and download it to create another isolated structure + fileGroupManager.addGroupForDownload(account2GroupKey, defaultFileGroup).get(); + fileGroupManager + .downloadFileGroup( + account2GroupKey, DownloadConditions.getDefaultInstance(), noCustomValidation()) + .get(); + DataFileGroupInternal storedFileGroupAccount2 = + fileGroupManager.getFileGroup(account2GroupKey, /* downloaded= */ true).get(); - fileGroupManager.verifyAndAttemptToRepairIsolatedFiles().get(); + Uri isolatedFileUriAccount2 = + fileGroupManager.getIsolatedFileUris(storedFileGroupAccount2).get(file); - assertThat(fileGroupManager.getFileGroup(defaultGroupKey, true).get()).isNotNull(); + // Check that both symlinks exist and point to the right file + assertThat(SymlinkUtil.readSymlink(context, isolatedFileUriAccount1)).isEqualTo(onDeviceUri); + assertThat(SymlinkUtil.readSymlink(context, isolatedFileUriAccount2)).isEqualTo(onDeviceUri); + + // Check that the symlinks are not equal to each other (since the roots are different); + assertThat(isolatedFileUriAccount1).isNotEqualTo(isolatedFileUriAccount2); + } + + @Test + public void testFileGroupManager_withIsolatedRoot_isolateForDifferentBuilds() throws Exception { + DataFileGroupInternal fileGroupBuild1 = defaultFileGroup.toBuilder().setBuildId(1).build(); + DataFileGroupInternal fileGroupBuild2 = defaultFileGroup.toBuilder().setBuildId(2).build(); + + sharedFilesMetadata.write(newFileKey, existingDownloadedSharedFile).get(); + + // Get the actual uri on device (this should be the same for both variants). + Uri onDeviceUri = fileGroupManager.getOnDeviceUri(file, fileGroupBuild1).get(); + // Actually write something to disk so the symlink points to something. + Void unused = + fileStorage.open(onDeviceUri, WriteByteArrayOpener.create("some content".getBytes(UTF_16))); + + // Add the first build and download it to create the isolated structure + fileGroupManager.addGroupForDownload(defaultGroupKey, fileGroupBuild1).get(); + fileGroupManager + .downloadFileGroup( + defaultGroupKey, DownloadConditions.getDefaultInstance(), noCustomValidation()) + .get(); + DataFileGroupInternal storedFileGroupBuild1 = + fileGroupManager.getFileGroup(defaultGroupKey, /* downloaded= */ true).get(); + + Uri isolatedFileUriBuild1 = + fileGroupManager.getIsolatedFileUris(storedFileGroupBuild1).get(file); + + // Add the second build and download it to create another isolated structure + fileGroupManager.addGroupForDownload(defaultGroupKey, fileGroupBuild2).get(); + fileGroupManager + .downloadFileGroup( + defaultGroupKey, DownloadConditions.getDefaultInstance(), noCustomValidation()) + .get(); + DataFileGroupInternal storedFileGroupBuild2 = + fileGroupManager.getFileGroup(defaultGroupKey, /* downloaded= */ true).get(); + + Uri isolatedFileUriBuild2 = + fileGroupManager.getIsolatedFileUris(storedFileGroupBuild2).get(file); + + // Check that both symlinks exist and point to the right file + assertThat(SymlinkUtil.readSymlink(context, isolatedFileUriBuild1)).isEqualTo(onDeviceUri); + assertThat(SymlinkUtil.readSymlink(context, isolatedFileUriBuild2)).isEqualTo(onDeviceUri); + + // Check that the symlinks are not equal to each other (since the roots are different); + assertThat(isolatedFileUriBuild1).isNotEqualTo(isolatedFileUriBuild2); + } + + @Test + public void testFileGroupManager_duplicateDownloadCalls_handlesIsolatedStructureCreation() + throws Exception { + writePendingFileGroup(defaultGroupKey, defaultFileGroup); + // Write an in progress file because we want to invoke the downloader and simulate a + // long-running download. This ensures that both download futures run their post-download + // workflow at the same time. + SharedFile existingInProgressSharedFile = + SharedFile.newBuilder() + .setFileStatus(FileStatus.DOWNLOAD_IN_PROGRESS) + .setFileName("fileName") + .setAndroidShared(false) + .build(); + sharedFilesMetadata.write(newFileKey, existingInProgressSharedFile).get(); + + Uri onDeviceUri = fileGroupManager.getOnDeviceUri(file, defaultFileGroup).get(); + // Actually write something to disk so the symlink points to something. + Void unused = + fileStorage.open(onDeviceUri, WriteByteArrayOpener.create("some content".getBytes(UTF_16))); + + // Start 2 downloads and wait for file download to start + ListenableFuture<?> downloadFuture1 = + fileGroupManager.downloadFileGroup( + defaultGroupKey, DownloadConditions.getDefaultInstance(), noCustomValidation()); + + ListenableFuture<?> downloadFuture2 = + fileGroupManager.downloadFileGroup( + defaultGroupKey, DownloadConditions.getDefaultInstance(), noCustomValidation()); + + blockingFileDownloader.waitForDownloadStarted(); + + // Both downloads should be waiting for the same file download, so finish downloading to get + // both performing the same post download process at the same time. + blockingFileDownloader.finishDownloading(); + + // Wait for both futures to complete. + downloadFuture1.get(); + downloadFuture2.get(); + + Uri isolatedFileUri = fileGroupManager.getIsolatedFileUris(defaultFileGroup).get(file); + + assertThat(SymlinkUtil.readSymlink(context, isolatedFileUri)).isEqualTo(onDeviceUri); } private void writePendingFileGroup(GroupKey key, DataFileGroupInternal group) throws Exception { diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/MddTestUtil.java b/javatests/com/google/android/libraries/mobiledatadownload/internal/MddTestUtil.java index 0cfa436..e3976c7 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/internal/MddTestUtil.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/MddTestUtil.java @@ -25,11 +25,6 @@ import android.content.Context; import android.os.Build.VERSION; import android.support.test.uiautomator.UiDevice; import android.util.Log; -import com.google.android.apps.common.testing.util.BackdoorTestUtil; -import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; -import com.google.mobiledatadownload.TransformProto.Transform; -import com.google.mobiledatadownload.TransformProto.Transforms; -import com.google.mobiledatadownload.TransformProto.ZipTransform; import com.google.mobiledatadownload.internal.MetadataProto.BaseFile; import com.google.mobiledatadownload.internal.MetadataProto.DataFile; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; @@ -38,6 +33,10 @@ import com.google.mobiledatadownload.internal.MetadataProto.DeltaFile.DiffDecode import com.google.mobiledatadownload.internal.MetadataProto.FileStatus; import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey; import com.google.mobiledatadownload.internal.MetadataProto.SharedFile; +import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; +import com.google.mobiledatadownload.TransformProto.Transform; +import com.google.mobiledatadownload.TransformProto.Transforms; +import com.google.mobiledatadownload.TransformProto.ZipTransform; import com.google.protobuf.MessageLite; import java.io.IOException; import java.util.Collections; @@ -103,7 +102,7 @@ public class MddTestUtil { DataFileGroupInternal.Builder dataFileGroupInternal = DataFileGroupInternal.newBuilder().setGroupName(fileGroupName); for (int i = 0; i < fileCount; ++i) { - dataFileGroupInternal.addFile(createSharedDataFile(fileGroupName, /* fileIndex = */ i)); + dataFileGroupInternal.addFile(createSharedDataFile(fileGroupName, /* fileIndex= */ i)); } return dataFileGroupInternal.build(); } @@ -252,24 +251,24 @@ public class MddTestUtil { } /** For API-level 19+, it moves the time forward by {@code timeInMillis} milliseconds. */ - public static void timeTravel(Context context, long timeInMillis) { - if (VERSION.SDK_INT == 18) { - throw new UnsupportedOperationException( - "Time travel does not work on API-level 18 - b/31132161. " - + "You need to disable this test on API-level 18. Example: cl/131498720"); - } - - final long timestampBeforeTravel = System.currentTimeMillis(); - if (!BackdoorTestUtil.advanceTime(context, timeInMillis)) { - // On some API levels (>23) the call returns false even if the time changed. Have a manual - // validation that the time changed instead. - if (VERSION.SDK_INT >= 23) { - assertThat(System.currentTimeMillis()).isAtLeast(timestampBeforeTravel + timeInMillis); - } else { - throw new IllegalStateException("Time Travel was not successful"); - } - } - } +// public static void timeTravel(Context context, long timeInMillis) { +// if (VERSION.SDK_INT == 18) { +// throw new UnsupportedOperationException( +// "Time travel does not work on API-level 18 - b/31132161. " +// + "You need to disable this test on API-level 18. Example: cl/131498720"); +// } +// +// final long timestampBeforeTravel = System.currentTimeMillis(); +// if (!BackdoorTestUtil.advanceTime(context, timeInMillis)) { +// // On some API levels (>23) the call returns false even if the time changed. Have a manual +// // validation that the time changed instead. +// if (VERSION.SDK_INT >= 23) { +// assertThat(System.currentTimeMillis()).isAtLeast(timestampBeforeTravel + timeInMillis); +// } else { +// throw new IllegalStateException("Time Travel was not successful"); +// } +// } +// } /** * @return the time (in seconds) that is n days from the current time diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/MobileDataDownloadManagerTest.java b/javatests/com/google/android/libraries/mobiledatadownload/internal/MobileDataDownloadManagerTest.java index 416a63a..549357b 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/internal/MobileDataDownloadManagerTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/MobileDataDownloadManagerTest.java @@ -16,17 +16,20 @@ package com.google.android.libraries.mobiledatadownload.internal; import static com.google.common.truth.Truth.assertThat; +import static com.google.common.util.concurrent.Futures.immediateFuture; import static com.google.common.util.concurrent.Futures.immediateVoidFuture; import static java.util.concurrent.TimeUnit.DAYS; import static org.junit.Assert.assertThrows; +import static org.junit.Assume.assumeTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isA; import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -38,6 +41,13 @@ import android.content.Context; import android.content.SharedPreferences; import android.net.Uri; import androidx.test.core.app.ApplicationProvider; +import com.google.mobiledatadownload.internal.MetadataProto.DataFile; +import com.google.mobiledatadownload.internal.MetadataProto.DataFile.ChecksumType; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; +import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions; +import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions.DeviceNetworkPolicy; +import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions.DeviceStoragePolicy; +import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; import com.google.android.libraries.mobiledatadownload.DownloadException; import com.google.android.libraries.mobiledatadownload.DownloadException.DownloadResultCode; import com.google.android.libraries.mobiledatadownload.FileSource; @@ -45,17 +55,18 @@ import com.google.android.libraries.mobiledatadownload.SilentFeedback; import com.google.android.libraries.mobiledatadownload.file.common.testing.TemporaryUri; import com.google.android.libraries.mobiledatadownload.internal.FileGroupManager.GroupDownloadStatus; import com.google.android.libraries.mobiledatadownload.internal.Migrations.FileKeyVersion; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup; import com.google.android.libraries.mobiledatadownload.internal.experimentation.DownloadStageManager; import com.google.android.libraries.mobiledatadownload.internal.experimentation.NoOpDownloadStageManager; import com.google.android.libraries.mobiledatadownload.internal.logging.EventLogger; import com.google.android.libraries.mobiledatadownload.internal.logging.FileGroupStatsLogger; import com.google.android.libraries.mobiledatadownload.internal.logging.LoggingStateStore; import com.google.android.libraries.mobiledatadownload.internal.logging.NetworkLogger; -import com.google.android.libraries.mobiledatadownload.internal.logging.NoOpLoggingState; import com.google.android.libraries.mobiledatadownload.internal.logging.StorageLogger; import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupUtil; import com.google.android.libraries.mobiledatadownload.internal.util.SharedPreferencesUtil; import com.google.android.libraries.mobiledatadownload.testing.FakeTimeSource; +import com.google.android.libraries.mobiledatadownload.testing.MddTestDependencies; import com.google.android.libraries.mobiledatadownload.testing.TestFlags; import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; @@ -64,20 +75,15 @@ import com.google.common.labs.concurrent.LabsFutures; import com.google.common.util.concurrent.AsyncFunction; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; import com.google.mobiledatadownload.TransformProto.CompressTransform; import com.google.mobiledatadownload.TransformProto.Transform; import com.google.mobiledatadownload.TransformProto.Transforms; import com.google.mobiledatadownload.TransformProto.ZipTransform; -import com.google.mobiledatadownload.internal.MetadataProto.DataFile; -import com.google.mobiledatadownload.internal.MetadataProto.DataFile.ChecksumType; -import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; -import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions; -import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions.DeviceNetworkPolicy; -import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions.DeviceStoragePolicy; -import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; import com.google.protobuf.ByteString; import java.io.IOException; import java.util.List; +import java.util.Random; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.concurrent.Executors; @@ -88,6 +94,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; import org.mockito.Captor; +import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; @@ -120,8 +127,12 @@ public class MobileDataDownloadManagerTest { private Context context; private MobileDataDownloadManager mddManager; private final TestFlags flags = new TestFlags(); - @Rule public final TemporaryUri tmpUri = new TemporaryUri(); - @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + + @Rule(order = 2) + public final TemporaryUri tmpUri = new TemporaryUri(); + + @Rule(order = 3) + public final MockitoRule mocks = MockitoJUnit.rule(); @Mock EventLogger mockLogger; @Mock SharedFileManager mockSharedFileManager; @@ -146,7 +157,9 @@ public class MobileDataDownloadManagerTest { this.testClock = new FakeTimeSource(); testClock.advance(1, DAYS); - loggingStateStore = new NoOpLoggingState(); + loggingStateStore = + MddTestDependencies.LoggingStateStoreImpl.SHARED_PREFERENCES.loggingStateStore( + context, Optional.absent(), testClock, CONTROL_EXECUTOR, new Random()); loggingStateStore.getAndResetDaysSinceLastMaintenance().get(); testClock.advance(1, DAYS); // The next call into logging state store will return 1 @@ -174,14 +187,21 @@ public class MobileDataDownloadManagerTest { // Enable migrations so that init doesn't run all migrations before each test. setMigrationState(MobileDataDownloadManager.MDD_MIGRATED_TO_OFFROAD, true); + when(mockSharedFileManager.init()).thenReturn(Futures.immediateFuture(true)); when(mockSharedFileManager.clear()).thenReturn(Futures.immediateFuture(null)); when(mockSharedFileManager.cancelDownload(any())).thenReturn(Futures.immediateFuture(null)); when(mockSharedFileManager.cancelDownloadAndClear()).thenReturn(Futures.immediateFuture(null)); + when(mockSharedFilesMetadata.init()).thenReturn(Futures.immediateFuture(true)); + when(mockSharedFilesMetadata.clear()).thenReturn(immediateVoidFuture()); + when(mockFileGroupsMetadata.init()).thenReturn(Futures.immediateFuture(null)); when(mockFileGroupsMetadata.clear()).thenReturn(Futures.immediateFuture(null)); - when(mockSharedFilesMetadata.clear()).thenReturn(Futures.immediateFuture(null)); + when(mockFileGroupsMetadata.getAllStaleGroups()) + .thenReturn(Futures.immediateFuture(ImmutableList.of())); + when(mockFileGroupsMetadata.getAllFreshGroups()) + .thenReturn(Futures.immediateFuture(ImmutableList.of())); } @After @@ -231,15 +251,21 @@ public class MobileDataDownloadManagerTest { // This tests that the default value of {allowed_readers, allowed_readers_enum} is to allow // access to all 1p google apps. DataFileGroupInternal dataFileGroup = MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 1); - when(mockFileGroupManager.addGroupForDownload(TEST_KEY, dataFileGroup)) + when(mockFileGroupManager.addGroupForDownload(eq(TEST_KEY), eq(dataFileGroup))) .thenReturn(Futures.immediateFuture(true)); - when(mockFileGroupManager.verifyPendingGroupDownloaded(eq(TEST_KEY), eq(dataFileGroup), any())) + when(mockFileGroupManager.getFileGroup(eq(TEST_KEY), anyBoolean())) + .thenReturn(immediateFuture(dataFileGroup)); + when(mockFileGroupManager.verifyGroupDownloaded( + eq(TEST_KEY), eq(dataFileGroup), anyBoolean(), any(), any())) .thenReturn(Futures.immediateFuture(GroupDownloadStatus.PENDING)); + when(mockFileGroupsMetadata.getAllFreshGroups()) + .thenReturn( + immediateFuture(ImmutableList.of(GroupKeyAndGroup.create(TEST_KEY, dataFileGroup)))); assertThat(mddManager.addGroupForDownload(TEST_KEY, dataFileGroup).get()).isTrue(); verify(mockFileGroupManager).addGroupForDownload(TEST_KEY, dataFileGroup); verify(mockFileGroupManager) - .verifyPendingGroupDownloaded(eq(TEST_KEY), eq(dataFileGroup), any()); + .verifyGroupDownloaded(eq(TEST_KEY), eq(dataFileGroup), anyBoolean(), any(), any()); verifyNoInteractions(mockLogger); assertThat(mddManager.addGroupForDownload(TEST_KEY, dataFileGroup).get()).isTrue(); @@ -264,13 +290,19 @@ public class MobileDataDownloadManagerTest { .build(); when(mockFileGroupManager.addGroupForDownload(TEST_KEY, dataFileGroup)) .thenReturn(Futures.immediateFuture(true)); - when(mockFileGroupManager.verifyPendingGroupDownloaded(eq(TEST_KEY), eq(dataFileGroup), any())) + when(mockFileGroupManager.getFileGroup(eq(TEST_KEY), anyBoolean())) + .thenReturn(immediateFuture(dataFileGroup)); + when(mockFileGroupManager.verifyGroupDownloaded( + eq(TEST_KEY), eq(dataFileGroup), anyBoolean(), any(), any())) .thenReturn(Futures.immediateFuture(GroupDownloadStatus.PENDING)); + when(mockFileGroupsMetadata.getAllFreshGroups()) + .thenReturn( + immediateFuture(ImmutableList.of(GroupKeyAndGroup.create(TEST_KEY, dataFileGroup)))); assertThat(mddManager.addGroupForDownload(TEST_KEY, dataFileGroup).get()).isTrue(); verify(mockFileGroupManager).addGroupForDownload(TEST_KEY, dataFileGroup); verify(mockFileGroupManager) - .verifyPendingGroupDownloaded(eq(TEST_KEY), eq(dataFileGroup), any()); + .verifyGroupDownloaded(eq(TEST_KEY), eq(dataFileGroup), anyBoolean(), any(), any()); verifyNoInteractions(mockLogger); assertThat(mddManager.addGroupForDownload(TEST_KEY, dataFileGroup).get()).isTrue(); @@ -283,13 +315,19 @@ public class MobileDataDownloadManagerTest { MddTestUtil.createFileGroupInternalWithDeltaFile(TEST_GROUP); when(mockFileGroupManager.addGroupForDownload(TEST_KEY, dataFileGroup)) .thenReturn(Futures.immediateFuture(true)); - when(mockFileGroupManager.verifyPendingGroupDownloaded(eq(TEST_KEY), eq(dataFileGroup), any())) + when(mockFileGroupManager.getFileGroup(eq(TEST_KEY), anyBoolean())) + .thenReturn(immediateFuture(dataFileGroup)); + when(mockFileGroupManager.verifyGroupDownloaded( + eq(TEST_KEY), eq(dataFileGroup), anyBoolean(), any(), any())) .thenReturn(Futures.immediateFuture(GroupDownloadStatus.PENDING)); + when(mockFileGroupsMetadata.getAllFreshGroups()) + .thenReturn( + immediateFuture(ImmutableList.of(GroupKeyAndGroup.create(TEST_KEY, dataFileGroup)))); assertThat(mddManager.addGroupForDownload(TEST_KEY, dataFileGroup).get()).isTrue(); verify(mockFileGroupManager).addGroupForDownload(TEST_KEY, dataFileGroup); verify(mockFileGroupManager) - .verifyPendingGroupDownloaded(eq(TEST_KEY), eq(dataFileGroup), any()); + .verifyGroupDownloaded(eq(TEST_KEY), eq(dataFileGroup), anyBoolean(), any(), any()); verifyNoInteractions(mockLogger); assertThat(mddManager.addGroupForDownload(TEST_KEY, dataFileGroup).get()).isTrue(); @@ -306,16 +344,22 @@ public class MobileDataDownloadManagerTest { .build(); when(mockFileGroupManager.addGroupForDownload(TEST_KEY, dataFileGroup)) .thenReturn(Futures.immediateFuture(true)); - when(mockFileGroupManager.verifyPendingGroupDownloaded(eq(TEST_KEY), eq(dataFileGroup), any())) + when(mockFileGroupManager.getFileGroup(eq(TEST_KEY), anyBoolean())) + .thenReturn(immediateFuture(dataFileGroup)); + when(mockFileGroupManager.verifyGroupDownloaded( + eq(TEST_KEY), eq(dataFileGroup), anyBoolean(), any(), any())) .thenReturn(Futures.immediateFuture(GroupDownloadStatus.DOWNLOADED)); + when(mockFileGroupsMetadata.getAllFreshGroups()) + .thenReturn( + immediateFuture(ImmutableList.of(GroupKeyAndGroup.create(TEST_KEY, dataFileGroup)))); assertThat(mddManager.addGroupForDownload(TEST_KEY, dataFileGroup).get()).isTrue(); verify(mockFileGroupManager).addGroupForDownload(TEST_KEY, dataFileGroup); verify(mockFileGroupManager) - .verifyPendingGroupDownloaded(eq(TEST_KEY), eq(dataFileGroup), any()); + .verifyGroupDownloaded(eq(TEST_KEY), eq(dataFileGroup), anyBoolean(), any(), any()); verify(mockLogger) .logEventSampled( - 0, + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, TEST_GROUP, /* fileGroupVersionNumber= */ 0, /* buildId= */ dataFileGroup.getBuildId(), @@ -378,15 +422,20 @@ public class MobileDataDownloadManagerTest { DataFileGroupInternal dataFileGroup = MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 1); when(mockFileGroupManager.addGroupForDownload(TEST_KEY, dataFileGroup)) .thenReturn(Futures.immediateFuture(true), Futures.immediateFuture(false)); - when(mockFileGroupManager.verifyPendingGroupDownloaded( - eq(TEST_KEY), any(DataFileGroupInternal.class), any())) + when(mockFileGroupManager.getFileGroup(eq(TEST_KEY), anyBoolean())) + .thenReturn(immediateFuture(dataFileGroup)); + when(mockFileGroupManager.verifyGroupDownloaded( + eq(TEST_KEY), eq(dataFileGroup), anyBoolean(), any(), any())) .thenReturn(Futures.immediateFuture(GroupDownloadStatus.PENDING)); + when(mockFileGroupsMetadata.getAllFreshGroups()) + .thenReturn( + immediateFuture(ImmutableList.of(GroupKeyAndGroup.create(TEST_KEY, dataFileGroup)))); assertThat(mddManager.addGroupForDownload(TEST_KEY, dataFileGroup).get()).isTrue(); assertThat(mddManager.addGroupForDownload(TEST_KEY, dataFileGroup).get()).isTrue(); verify(mockFileGroupManager, times(2)).addGroupForDownload(TEST_KEY, dataFileGroup); verify(mockFileGroupManager, times(1)) - .verifyPendingGroupDownloaded(eq(TEST_KEY), eq(dataFileGroup), any()); + .verifyGroupDownloaded(eq(TEST_KEY), eq(dataFileGroup), anyBoolean(), any(), any()); verifyNoInteractions(mockExpirationHandler); verifyNoInteractions(mockLogger); } @@ -404,7 +453,7 @@ public class MobileDataDownloadManagerTest { verify(mockLogger) .logEventSampled( - 0, + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, "", /* fileGroupVersionNumber= */ 0, /* buildId= */ dataFileGroup.getBuildId(), @@ -429,9 +478,14 @@ public class MobileDataDownloadManagerTest { when(mockFileGroupManager.addGroupForDownload(eq(TEST_KEY), dataFileGroupCaptor.capture())) .thenReturn(Futures.immediateFuture(true)); - when(mockFileGroupManager.verifyPendingGroupDownloaded( - eq(TEST_KEY), any(DataFileGroupInternal.class), any())) + when(mockFileGroupManager.getFileGroup(eq(TEST_KEY), anyBoolean())) + .thenReturn(Futures.immediateFuture(dataFileGroup)); + when(mockFileGroupManager.verifyGroupDownloaded( + eq(TEST_KEY), eq(dataFileGroup), anyBoolean(), any(), any())) .thenReturn(Futures.immediateFuture(GroupDownloadStatus.PENDING)); + when(mockFileGroupsMetadata.getAllFreshGroups()) + .thenReturn( + immediateFuture(ImmutableList.of(GroupKeyAndGroup.create(TEST_KEY, dataFileGroup)))); assertThat(mddManager.addGroupForDownload(TEST_KEY, dataFileGroup).get()).isTrue(); verifyNoInteractions(mockLogger); @@ -468,9 +522,14 @@ public class MobileDataDownloadManagerTest { when(mockFileGroupManager.addGroupForDownload(eq(TEST_KEY), dataFileGroupCaptor.capture())) .thenReturn(Futures.immediateFuture(true)); - when(mockFileGroupManager.verifyPendingGroupDownloaded( - eq(TEST_KEY), any(DataFileGroupInternal.class), any())) + when(mockFileGroupManager.getFileGroup(eq(TEST_KEY), anyBoolean())) + .thenReturn(immediateFuture(dataFileGroup)); + when(mockFileGroupManager.verifyGroupDownloaded( + eq(TEST_KEY), eq(dataFileGroup), anyBoolean(), any(), any())) .thenReturn(Futures.immediateFuture(GroupDownloadStatus.PENDING)); + when(mockFileGroupsMetadata.getAllFreshGroups()) + .thenReturn( + immediateFuture(ImmutableList.of(GroupKeyAndGroup.create(TEST_KEY, dataFileGroup)))); assertThat(mddManager.addGroupForDownload(TEST_KEY, dataFileGroup).get()).isTrue(); verifyNoInteractions(mockLogger); @@ -498,7 +557,11 @@ public class MobileDataDownloadManagerTest { assertThat(mddManager.addGroupForDownload(TEST_KEY, dataFileGroup).get()).isFalse(); verify(mockLogger) .logEventSampled( - 0, TEST_GROUP, /* fileGroupVersionNumber= */ 0, /* buildId= */ 0, /* variantId= */ ""); + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + TEST_GROUP, + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); verifyNoInteractions(mockFileGroupManager); } @@ -519,8 +582,14 @@ public class MobileDataDownloadManagerTest { when(mockFileGroupManager.addGroupForDownload(eq(TEST_KEY), any())) .thenReturn(Futures.immediateFuture(true)); - when(mockFileGroupManager.verifyPendingGroupDownloaded(eq(TEST_KEY), any(), any())) - .thenReturn(Futures.immediateFuture(GroupDownloadStatus.DOWNLOADED)); + when(mockFileGroupManager.getFileGroup(eq(TEST_KEY), anyBoolean())) + .thenReturn(immediateFuture(sideloadedGroup)); + when(mockFileGroupManager.verifyGroupDownloaded( + eq(TEST_KEY), eq(sideloadedGroup), anyBoolean(), any(), any())) + .thenReturn(Futures.immediateFuture(GroupDownloadStatus.PENDING)); + when(mockFileGroupsMetadata.getAllFreshGroups()) + .thenReturn( + immediateFuture(ImmutableList.of(GroupKeyAndGroup.create(TEST_KEY, sideloadedGroup)))); { // Force sideloading off @@ -645,14 +714,23 @@ public class MobileDataDownloadManagerTest { public void testGetDataFileUri() throws Exception { DataFileGroupInternal dataFileGroup = MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 2); - when(mockFileGroupManager.getOnDeviceUri(dataFileGroup.getFile(0), dataFileGroup)) - .thenReturn(Futures.immediateFuture(fileUri1)); - when(mockFileGroupManager.getOnDeviceUri(dataFileGroup.getFile(1), dataFileGroup)) - .thenReturn(Futures.immediateFuture(fileUri2)); + when(mockFileGroupManager.getOnDeviceUris(dataFileGroup)) + .thenReturn( + Futures.immediateFuture( + ImmutableMap.of( + dataFileGroup.getFile(0), fileUri1, dataFileGroup.getFile(1), fileUri2))); - assertThat(mddManager.getDataFileUri(dataFileGroup.getFile(0), dataFileGroup).get()) + assertThat( + mddManager + .getDataFileUri( + dataFileGroup.getFile(0), dataFileGroup, /* verifyIsolatedStructure= */ true) + .get()) .isEqualTo(fileUri1); - assertThat(mddManager.getDataFileUri(dataFileGroup.getFile(1), dataFileGroup).get()) + assertThat( + mddManager + .getDataFileUri( + dataFileGroup.getFile(1), dataFileGroup, /* verifyIsolatedStructure= */ true) + .get()) .isEqualTo(fileUri2); } @@ -670,14 +748,23 @@ public class MobileDataDownloadManagerTest { .setFile(0, dataFileGroup.getFile(0).toBuilder().setReadTransforms(compressTransform)) .build(); - when(mockFileGroupManager.getOnDeviceUri(dataFileGroup.getFile(0), dataFileGroup)) - .thenReturn(Futures.immediateFuture(fileUri1)); - when(mockFileGroupManager.getOnDeviceUri(dataFileGroup.getFile(1), dataFileGroup)) - .thenReturn(Futures.immediateFuture(fileUri2)); + when(mockFileGroupManager.getOnDeviceUris(dataFileGroup)) + .thenReturn( + Futures.immediateFuture( + ImmutableMap.of( + dataFileGroup.getFile(0), fileUri1, dataFileGroup.getFile(1), fileUri2))); - assertThat(mddManager.getDataFileUri(dataFileGroup.getFile(0), dataFileGroup).get()) + assertThat( + mddManager + .getDataFileUri( + dataFileGroup.getFile(0), dataFileGroup, /* verifyIsolatedStructure= */ true) + .get()) .isEqualTo(fileUri1.buildUpon().encodedFragment("transform=compress").build()); - assertThat(mddManager.getDataFileUri(dataFileGroup.getFile(1), dataFileGroup).get()) + assertThat( + mddManager + .getDataFileUri( + dataFileGroup.getFile(1), dataFileGroup, /* verifyIsolatedStructure= */ true) + .get()) .isEqualTo(fileUri2); } @@ -695,13 +782,18 @@ public class MobileDataDownloadManagerTest { FileGroupUtil.getIsolatedFileUri( context, Optional.absent(), relativePathFile, testFileGroup); - when(mockFileGroupManager.getOnDeviceUri(testFileGroup.getFile(0), testFileGroup)) - .thenReturn(Futures.immediateFuture(fileUri1)); - when(mockFileGroupManager.getAndVerifyIsolatedFileUri( - fileUri1, relativePathFile, testFileGroup)) - .thenReturn(symlinkedUri); + when(mockFileGroupManager.getOnDeviceUris(testFileGroup)) + .thenReturn(Futures.immediateFuture(ImmutableMap.of(testFileGroup.getFile(0), fileUri1))); + when(mockFileGroupManager.getIsolatedFileUris(testFileGroup)) + .thenReturn(ImmutableMap.of(testFileGroup.getFile(0), symlinkedUri)); + when(mockFileGroupManager.verifyIsolatedFileUris(any(), any())) + .thenReturn(ImmutableMap.of(testFileGroup.getFile(0), symlinkedUri)); - assertThat(mddManager.getDataFileUri(relativePathFile, testFileGroup).get()) + assertThat( + mddManager + .getDataFileUri( + relativePathFile, testFileGroup, /* verifyIsolatedStructure= */ true) + .get()) .isEqualTo(symlinkedUri); } @@ -715,13 +807,17 @@ public class MobileDataDownloadManagerTest { .addFile(relativePathFile) .build(); - when(mockFileGroupManager.getOnDeviceUri(testFileGroup.getFile(0), testFileGroup)) - .thenReturn(Futures.immediateFuture(fileUri1)); - when(mockFileGroupManager.getAndVerifyIsolatedFileUri( - fileUri1, relativePathFile, testFileGroup)) - .thenThrow(new IOException("test failure")); + when(mockFileGroupManager.getOnDeviceUris(testFileGroup)) + .thenReturn(Futures.immediateFuture(ImmutableMap.of(testFileGroup.getFile(0), fileUri1))); + when(mockFileGroupManager.getIsolatedFileUris(testFileGroup)).thenReturn(ImmutableMap.of()); + when(mockFileGroupManager.verifyIsolatedFileUris(any(), any())).thenReturn(ImmutableMap.of()); - assertThat(mddManager.getDataFileUri(relativePathFile, testFileGroup).get()).isNull(); + assertThat( + mddManager + .getDataFileUri( + relativePathFile, testFileGroup, /* verifyIsolatedStructure= */ true) + .get()) + .isNull(); } @Test @@ -867,7 +963,7 @@ public class MobileDataDownloadManagerTest { mddManager.downloadAllPendingGroups(true, noCustomValidation()).get(); - verify(mockLogger).logEventSampled(0); + verify(mockLogger).logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); verify(mockFileGroupManager).scheduleAllPendingGroupsForDownload(eq(true), any()); verifyNoMoreInteractions(mockLogger); } @@ -880,12 +976,14 @@ public class MobileDataDownloadManagerTest { mddManager.verifyAllPendingGroups(noCustomValidation()).get(); verify(mockFileGroupManager).verifyAllPendingGroupsDownloaded(any()); - verify(mockLogger).logEventSampled(0); + verify(mockLogger).logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); verifyNoMoreInteractions(mockLogger); } @Test public void testMaintenance_mddFileExpiration() throws Exception { + assumeTrue(flags.mddEnableGarbageCollection()); + setupMaintenanceTasks(); mddManager.maintenance().get(); @@ -894,8 +992,18 @@ public class MobileDataDownloadManagerTest { verify(mockExpirationHandler).updateExpiration(); - verify(mockFileGroupStatsLogger).log(anyInt()); - verify(mockLogger).logEventSampled(0); + verify(mockFileGroupStatsLogger).log(DEFAULT_DAYS_SINCE_LAST_LOG); + verify(mockLogger).logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); + } + + @Test + public void testMaintenance_gcFlagControlsGcDuringMaintenance() throws Exception { + setupMaintenanceTasks(); + flags.mddEnableGarbageCollection = Optional.of(false); + + mddManager.maintenance().get(); + + verify(mockExpirationHandler, never()).updateExpiration(); } @Test @@ -904,7 +1012,7 @@ public class MobileDataDownloadManagerTest { mddManager.maintenance().get(); - verify(mockFileGroupStatsLogger).log(anyInt()); + verify(mockStorageLogger).logStorageStats(DEFAULT_DAYS_SINCE_LAST_LOG); } @Test @@ -955,11 +1063,14 @@ public class MobileDataDownloadManagerTest { } void setupMaintenanceTasks() { + flags.enableDaysSinceLastMaintenanceTracking = Optional.of(true); - when(mockStorageLogger.logStorageStats(anyInt())).thenReturn(Futures.immediateVoidFuture()); + when(mockStorageLogger.logStorageStats(DEFAULT_DAYS_SINCE_LAST_LOG)) + .thenReturn(Futures.immediateVoidFuture()); when(mockExpirationHandler.updateExpiration()).thenReturn(Futures.immediateVoidFuture()); - when(mockFileGroupStatsLogger.log(anyInt())).thenReturn(Futures.immediateVoidFuture()); + when(mockFileGroupStatsLogger.log(DEFAULT_DAYS_SINCE_LAST_LOG)) + .thenReturn(Futures.immediateVoidFuture()); when(mockNetworkLogger.log()).thenReturn(Futures.immediateVoidFuture()); when(mockFileGroupManager.logAndDeleteForMissingSharedFiles()) .thenReturn(Futures.immediateVoidFuture()); @@ -974,6 +1085,15 @@ public class MobileDataDownloadManagerTest { } @Test + public void testRemoveExpiredGroupsAndFiles() throws Exception { + setupMaintenanceTasks(); + + mddManager.removeExpiredGroupsAndFiles().get(); + + verify(mockExpirationHandler).updateExpiration(); + } + + @Test public void testClear() throws Exception { mddManager.clear().get(); @@ -1001,7 +1121,7 @@ public class MobileDataDownloadManagerTest { mddManager.checkResetTrigger().get(); verify(mockSharedFileManager).cancelDownloadAndClear(); - verify(mockLogger).logEventSampled(0); + verify(mockLogger).logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); // saved reset value should be set to 2 checkSavedResetValue(2); verifyNoMoreInteractions(mockLogger); @@ -1016,7 +1136,7 @@ public class MobileDataDownloadManagerTest { // The second check should have no effect - clear should only be called once. mddManager.checkResetTrigger().get(); verify(mockSharedFileManager).cancelDownloadAndClear(); - verify(mockLogger).logEventSampled(0); + verify(mockLogger).logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); // saved reset value should be set to 2 checkSavedResetValue(2); verifyNoMoreInteractions(mockLogger); @@ -1035,12 +1155,41 @@ public class MobileDataDownloadManagerTest { mddManager.checkResetTrigger().get(); verify(mockSharedFileManager, times(2)).cancelDownloadAndClear(); - verify(mockLogger, times(2)).logEventSampled(0); + verify(mockLogger, times(2)).logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); // saved reset value should be set to 2 checkSavedResetValue(3); verifyNoMoreInteractions(mockLogger); } + @Test + public void testClear_resetsExperimentIds() throws Exception { + flags.enableDownloadStageExperimentIdPropagation = Optional.of(true); + + long buildId = 999L; + int experimentId = 12345; + DataFileGroupInternal dataFileGroup = + MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 2).toBuilder() + .setBuildId(buildId) + .build(); + + when(mockFileGroupsMetadata.getAllFreshGroups()) + .thenReturn( + immediateFuture( + ImmutableList.of( + GroupKeyAndGroup.create( + GroupKey.newBuilder().setGroupName(TEST_GROUP).build(), dataFileGroup)))); + + when(mockFileGroupsMetadata.getAllStaleGroups()) + .thenReturn(immediateFuture(ImmutableList.of())); + + mddManager.clear().get(); + + InOrder inOrder = inOrder(mockFileGroupsMetadata); + + inOrder.verify(mockFileGroupsMetadata).getAllFreshGroups(); + inOrder.verify(mockFileGroupsMetadata).clear(); + } + private void setMigrationState(String key, boolean value) { SharedPreferences sharedPreferences = SharedPreferencesUtil.getSharedPreferences( diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/SharedFileManagerTest.java b/javatests/com/google/android/libraries/mobiledatadownload/internal/SharedFileManagerTest.java index 30d5682..d8c87f1 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/internal/SharedFileManagerTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/SharedFileManagerTest.java @@ -17,6 +17,7 @@ package com.google.android.libraries.mobiledatadownload.internal; import static com.google.android.libraries.mobiledatadownload.internal.SharedFileManager.MDD_SHARED_FILE_MANAGER_METADATA; import static com.google.common.truth.Truth.assertThat; +import static com.google.common.util.concurrent.Futures.immediateFuture; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyList; @@ -33,6 +34,16 @@ import android.content.SharedPreferences; import android.net.Uri; import android.os.Build; import androidx.test.core.app.ApplicationProvider; +import com.google.mobiledatadownload.internal.MetadataProto.DataFile; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal.AllowedReaders; +import com.google.mobiledatadownload.internal.MetadataProto.DeltaFile; +import com.google.mobiledatadownload.internal.MetadataProto.DeltaFile.DiffDecoder; +import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions; +import com.google.mobiledatadownload.internal.MetadataProto.FileStatus; +import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; +import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey; +import com.google.mobiledatadownload.internal.MetadataProto.SharedFile; import com.google.android.libraries.mobiledatadownload.DownloadException; import com.google.android.libraries.mobiledatadownload.DownloadException.DownloadResultCode; import com.google.android.libraries.mobiledatadownload.FileSource; @@ -57,16 +68,7 @@ import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.MoreExecutors; -import com.google.mobiledatadownload.internal.MetadataProto.DataFile; -import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; -import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal.AllowedReaders; -import com.google.mobiledatadownload.internal.MetadataProto.DeltaFile; -import com.google.mobiledatadownload.internal.MetadataProto.DeltaFile.DiffDecoder; -import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions; -import com.google.mobiledatadownload.internal.MetadataProto.FileStatus; -import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; -import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey; -import com.google.mobiledatadownload.internal.MetadataProto.SharedFile; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; import com.google.protobuf.ByteString; import java.io.File; import java.io.FileOutputStream; @@ -119,6 +121,7 @@ public class SharedFileManagerTest { private static final String TEST_GROUP = "test-group"; private static final int VERSION_NUMBER = 7; private static final long BUILD_ID = 0; + private static final String VARIANT_ID = ""; private static final DataFileGroupInternal FILE_GROUP = MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 1).toBuilder() .setFileGroupVersionNumber(VERSION_NUMBER) @@ -136,6 +139,7 @@ public class SharedFileManagerTest { private File privateDirectory; private Optional<DeltaDecoder> deltaDecoder; private final TestFlags flags = new TestFlags(); + @Mock SilentFeedback mockSilentFeedback; @Mock MddFileDownloader mockDownloader; @Mock DownloadProgressMonitor mockDownloadMonitor; @@ -156,6 +160,8 @@ public class SharedFileManagerTest { Arrays.asList(AndroidFileBackend.builder(context).build(), mockBackend), ImmutableList.of(new CompressTransform())); + when(fileGroupsMetadata.read(any())).thenReturn(immediateFuture(null)); + sharedFilesMetadata = new SharedPreferencesSharedFilesMetadata( context, mockSilentFeedback, Optional.absent(), flags); @@ -290,7 +296,7 @@ public class SharedFileManagerTest { // The partial download file should be deleted assertThat(onDeviceFile.exists()).isTrue(); - verify(mockDownloader).stopDownloading(uri); + verify(mockDownloader).stopDownloading(newFileKey.getChecksum(), uri); } @Test @@ -307,6 +313,7 @@ public class SharedFileManagerTest { Uri fileUri = sfm.getOnDeviceUri(newFileKey).get(); when(fileGroupsMetadata.read(GROUP_KEY)).thenReturn(Futures.immediateFuture(FILE_GROUP)); when(mockDownloader.startCopying( + eq(newFileKey.getChecksum()), eq(fileUri), eq(file.getUrlToDownload()), eq(file.getByteSize()), @@ -339,7 +346,8 @@ public class SharedFileManagerTest { sfm.startImport(GROUP_KEY, file, newFileKey, DOWNLOAD_CONDITIONS, inlineSource).get(); onDeviceFile.delete(); - verify(mockDownloader, times(0)).startCopying(any(), any(), anyInt(), any(), any(), any()); + verify(mockDownloader, times(0)) + .startCopying(any(), any(), any(), anyInt(), any(), any(), any()); } @Test @@ -400,9 +408,11 @@ public class SharedFileManagerTest { when(fileGroupsMetadata.read(GROUP_KEY)).thenReturn(Futures.immediateFuture(FILE_GROUP)); when(mockDownloader.startDownloading( + eq(newFileKey.getChecksum()), eq(GROUP_KEY), eq(VERSION_NUMBER), eq(BUILD_ID), + eq(VARIANT_ID), eq(fileUri), eq(file.getUrlToDownload()), eq(file.getByteSize()), @@ -418,7 +428,7 @@ public class SharedFileManagerTest { newFileKey, DOWNLOAD_CONDITIONS, TRAFFIC_TAG, - /*extraHttpHeaders = */ ImmutableList.of()) + /* extraHttpHeaders= */ ImmutableList.of()) .get(); SharedFile sharedFile = sharedFilesMetadata.read(newFileKey).get(); @@ -448,7 +458,7 @@ public class SharedFileManagerTest { // The file should not be deleted by the SFM because deletion is handled by ExpirationHandler. assertThat(onDeviceFile.exists()).isTrue(); - verify(mockDownloader).stopDownloading(uri); + verify(mockDownloader).stopDownloading(newFileKey.getChecksum(), uri); } @Test @@ -471,7 +481,7 @@ public class SharedFileManagerTest { newFileKey, DOWNLOAD_CONDITIONS, TRAFFIC_TAG, - /* extraHttpHeaders = */ ImmutableList.of()) + /* extraHttpHeaders= */ ImmutableList.of()) .get()); assertThat(ex).hasCauseThat().isInstanceOf(DownloadException.class); DownloadException dex = (DownloadException) ex.getCause(); @@ -495,7 +505,7 @@ public class SharedFileManagerTest { newFileKey, DOWNLOAD_CONDITIONS, TRAFFIC_TAG, - /*extraHttpHeaders = */ ImmutableList.of()) + /* extraHttpHeaders= */ ImmutableList.of()) .get()); assertThat(ex).hasCauseThat().isInstanceOf(DownloadException.class); assertThat(ex).hasMessageThat().contains("SHARED_FILE_NOT_FOUND_ERROR"); @@ -513,9 +523,11 @@ public class SharedFileManagerTest { Uri fileUri = sfm.getOnDeviceUri(newFileKey).get(); when(fileGroupsMetadata.read(GROUP_KEY)).thenReturn(Futures.immediateFuture(FILE_GROUP)); when(mockDownloader.startDownloading( + eq(newFileKey.getChecksum()), eq(GROUP_KEY), eq(VERSION_NUMBER), eq(BUILD_ID), + eq(VARIANT_ID), eq(fileUri), eq(file.getUrlToDownload()), eq(file.getByteSize()), @@ -531,7 +543,7 @@ public class SharedFileManagerTest { newFileKey, DOWNLOAD_CONDITIONS, TRAFFIC_TAG, - /* extraHttpHeaders = */ ImmutableList.of()) + /* extraHttpHeaders= */ ImmutableList.of()) .get(); SharedFile sharedFile = sharedFilesMetadata.read(newFileKey).get(); @@ -555,7 +567,7 @@ public class SharedFileManagerTest { newFileKey, DOWNLOAD_CONDITIONS, TRAFFIC_TAG, - /* extraHttpHeaders = */ ImmutableList.of()) + /* extraHttpHeaders= */ ImmutableList.of()) .get(); onDeviceFile.delete(); @@ -751,7 +763,7 @@ public class SharedFileManagerTest { assertThat(onDevicePublicFile.exists()).isFalse(); verify(mockBackend, never()).deleteFile(any()); - verify(eventLogger, never()).logEventSampled(0); + verify(eventLogger, never()).logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); } @Test @@ -761,9 +773,9 @@ public class SharedFileManagerTest { // Create three files, one downloaded, the other currently being downloaded and one shared with // the Android Blob Sharing Service. - DataFile downloadedFile = MddTestUtil.createDataFile("file", /* fileIndex = */ 0); - DataFile registeredFile = MddTestUtil.createDataFile("registered-file", /* fileIndex = */ 1); - DataFile sharedFile = MddTestUtil.createSharedDataFile("shared-file", /* fileIndex = */ 2); + DataFile downloadedFile = MddTestUtil.createDataFile("file", /* fileIndex= */ 0); + DataFile registeredFile = MddTestUtil.createDataFile("registered-file", /* fileIndex= */ 1); + DataFile sharedFile = MddTestUtil.createSharedDataFile("shared-file", /* fileIndex= */ 2); NewFileKey downloadedKey = SharedFilesMetadata.createKeyFromDataFile(downloadedFile, AllowedReaders.ALL_GOOGLE_APPS); @@ -799,7 +811,7 @@ public class SharedFileManagerTest { assertThat(onDevicePublicFile.exists()).isFalse(); verify(mockBackend).deleteFile(allLeasesUri); - verify(eventLogger).logEventSampled(0); + verify(eventLogger).logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); } @Test @@ -839,12 +851,12 @@ public class SharedFileManagerTest { mockSilentFeedback, /* instanceId= */ Optional.absent(), false); - verify(mockDownloader).stopDownloading(onDeviceUri); + verify(mockDownloader).stopDownloading(registeredKey.getChecksum(), onDeviceUri); } @Test public void testGetSharedFile() throws Exception { - DataFile file = MddTestUtil.createDataFile("fileId", /* fileIndex = */ 0); + DataFile file = MddTestUtil.createDataFile("fileId", /* fileIndex= */ 0); NewFileKey newFileKey = SharedFilesMetadata.createKeyFromDataFile(file, AllowedReaders.ALL_GOOGLE_APPS); diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/SharedFilesMetadataTest.java b/javatests/com/google/android/libraries/mobiledatadownload/internal/SharedFilesMetadataTest.java index ec5e139..ce73c04 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/internal/SharedFilesMetadataTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/SharedFilesMetadataTest.java @@ -19,10 +19,17 @@ import static com.google.common.truth.Truth.assertThat; import android.content.Context; import android.content.SharedPreferences; +import android.net.Uri; import androidx.test.core.app.ApplicationProvider; +import com.google.mobiledatadownload.internal.MetadataProto.DataFile; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal.AllowedReaders; +import com.google.mobiledatadownload.internal.MetadataProto.FileStatus; +import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey; +import com.google.mobiledatadownload.internal.MetadataProto.SharedFile; import com.google.android.libraries.mobiledatadownload.SilentFeedback; import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; import com.google.android.libraries.mobiledatadownload.file.backends.AndroidFileBackend; +import com.google.android.libraries.mobiledatadownload.file.backends.AndroidUri; import com.google.android.libraries.mobiledatadownload.internal.Migrations.FileKeyVersion; import com.google.android.libraries.mobiledatadownload.internal.logging.EventLogger; import com.google.android.libraries.mobiledatadownload.internal.util.SharedFilesMetadataUtil; @@ -34,11 +41,7 @@ import com.google.common.util.concurrent.MoreExecutors; import com.google.mobiledatadownload.TransformProto.CompressTransform; import com.google.mobiledatadownload.TransformProto.Transform; import com.google.mobiledatadownload.TransformProto.Transforms; -import com.google.mobiledatadownload.internal.MetadataProto.DataFile; -import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal.AllowedReaders; -import com.google.mobiledatadownload.internal.MetadataProto.FileStatus; -import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey; -import com.google.mobiledatadownload.internal.MetadataProto.SharedFile; +import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -82,8 +85,11 @@ public class SharedFilesMetadataTest { private SynchronousFileStorage storage; private Context context; private SharedFilesMetadata sharedFilesMetadata; + private Uri diagnosticUri; + private Uri destinationUri; private final TestFlags flags = new TestFlags(); + @Mock SilentFeedback mockSilentFeedback; @Mock EventLogger mockLogger; @@ -104,6 +110,17 @@ public class SharedFilesMetadataTest { storage = new SynchronousFileStorage(Arrays.asList(AndroidFileBackend.builder(context).build())); + destinationUri = + AndroidUri.builder(context) + .setPackage(context.getPackageName()) + .setRelativePath("dest.pb") + .build(); + diagnosticUri = + AndroidUri.builder(context) + .setPackage(context.getPackageName()) + .setRelativePath("diag.pb") + .build(); + SharedPreferencesSharedFilesMetadata sharedPreferencesMetadata = new SharedPreferencesSharedFilesMetadata(context, mockSilentFeedback, instanceId, flags); @@ -118,7 +135,13 @@ public class SharedFilesMetadataTest { } @After - public void tearDown() throws Exception { + public void tearDown() throws InterruptedException, ExecutionException, IOException { + if (storage.exists(diagnosticUri)) { + storage.deleteFile(diagnosticUri); + } + if (storage.exists(destinationUri)) { + storage.deleteFile(destinationUri); + } synchronized (SharedPreferencesSharedFilesMetadata.class) { sharedFilesMetadata.clear().get(); assertThat( @@ -314,6 +337,7 @@ public class SharedFilesMetadataTest { @Test public void testNoMigrate_corruptedMetadata() throws InterruptedException, ExecutionException { flags.fileKeyVersion = Optional.of(FileKeyVersion.USE_CHECKSUM_ONLY.value); + Migrations.setCurrentVersion(context, FileKeyVersion.USE_CHECKSUM_ONLY); // Create two files, one downloaded and the other currently being downloaded. diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/BUILD new file mode 100644 index 0000000..941463b --- /dev/null +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/BUILD @@ -0,0 +1,180 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +load("//javatests/com/google/android/libraries/mobiledatadownload:test_defs.bzl", "mdd_local_test") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//:__subpackages__"], + licenses = ["notice"], +) + +mdd_local_test( + name = "FileGroupStatsLoggerTest", + srcs = ["FileGroupStatsLoggerTest.java"], + test_class = "com.google.android.libraries.mobiledatadownload.internal.logging.FileGroupStatsLoggerTest", + deps = [ + "//java/com/google/android/libraries/mobiledatadownload/internal:FileGroupManager", + "//java/com/google/android/libraries/mobiledatadownload/internal:FileGroupsMetadata", + "//java/com/google/android/libraries/mobiledatadownload/internal/collect", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:FileGroupStatsLogger", + "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "//java/com/google/android/libraries/mobiledatadownload/internal/util:FileGroupUtil", + "//javatests/com/google/android/libraries/mobiledatadownload/internal:MddTestUtil", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", + "@com_google_guava_guava", + "@mockito", + "@truth", + ], +) + +mdd_local_test( + name = "StorageLoggerTest", + srcs = ["StorageLoggerTest.java"], + test_class = "com.google.android.libraries.mobiledatadownload.internal.logging.StorageLoggerTest", + deps = [ + "//java/com/google/android/libraries/mobiledatadownload:SilentFeedback", + "//java/com/google/android/libraries/mobiledatadownload/file", + "//java/com/google/android/libraries/mobiledatadownload/file/spi", + "//java/com/google/android/libraries/mobiledatadownload/internal:FileGroupsMetadata", + "//java/com/google/android/libraries/mobiledatadownload/internal:MddConstants", + "//java/com/google/android/libraries/mobiledatadownload/internal:SharedFileManager", + "//java/com/google/android/libraries/mobiledatadownload/internal:SharedFilesMetadata", + "//java/com/google/android/libraries/mobiledatadownload/internal/collect", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:StorageLogger", + "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "//java/com/google/android/libraries/mobiledatadownload/internal/util:DirectoryUtil", + "//javatests/com/google/android/libraries/mobiledatadownload/internal:MddTestUtil", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFlags", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", + "@com_google_auto_value", + "@com_google_guava_guava", + "@mockito", + "@truth", + ], +) + +mdd_local_test( + name = "NetworkLoggerTest", + srcs = ["NetworkLoggerTest.java"], + test_class = "com.google.android.libraries.mobiledatadownload.internal.logging.NetworkLoggerTest", + deps = [ + "//java/com/google/android/libraries/mobiledatadownload:Flags", + "//java/com/google/android/libraries/mobiledatadownload/file", + "//java/com/google/android/libraries/mobiledatadownload/file/backends:file", + "//java/com/google/android/libraries/mobiledatadownload/file/common/testing", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LoggingStateStore", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:NetworkLogger", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:SharedPreferencesLoggingState", + "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:FakeTimeSource", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:MddTestDependencies", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFlags", + "//proto:logs_java_proto_lite", + "@androidx_test", + "@com_google_guava_guava", + "@mockito", + "@truth", + ], +) + +mdd_local_test( + name = "MddEventLoggerTest", + srcs = ["MddEventLoggerTest.java"], + test_class = "com.google.android.libraries.mobiledatadownload.internal.logging.MddEventLoggerTest", + deps = [ + "//java/com/google/android/libraries/mobiledatadownload:Flags", + "//java/com/google/android/libraries/mobiledatadownload:Logger", + "//java/com/google/android/libraries/mobiledatadownload/file", + "//java/com/google/android/libraries/mobiledatadownload/file/backends:android", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogSampler", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:MddEventLogger", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:SharedPreferencesLoggingState", + "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:FakeTimeSource", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:MddTestDependencies", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFlags", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", + "@com_google_guava_guava", + "@mockito", + "@robolectric", + ], +) + +mdd_local_test( + name = "LoggingStateStoreTest", + srcs = ["LoggingStateStoreTest.java"], + test_class = "com.google.android.libraries.mobiledatadownload.internal.logging.LoggingStateStoreTest", + deps = [ + "//java/com/google/android/libraries/mobiledatadownload/file", + "//java/com/google/android/libraries/mobiledatadownload/file/common/testing", + "//java/com/google/android/libraries/mobiledatadownload/file/common/testing:fake_file_backend", + "//java/com/google/android/libraries/mobiledatadownload/internal:MddConstants", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LoggingStateStore", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:SharedPreferencesLoggingState", + "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "//java/com/google/android/libraries/mobiledatadownload/internal/util:SharedPreferencesUtil", + "//java/com/google/protobuf/util:time_lite", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:FakeTimeSource", + "@androidx_test", + "@com_google_guava_guava", + "@truth", + ], +) + +mdd_local_test( + name = "LogSamplerTest", + srcs = ["LogSamplerTest.java"], + test_class = "com.google.android.libraries.mobiledatadownload.internal.logging.LogSamplerTest", + deps = [ + "//java/com/google/android/libraries/mobiledatadownload:Flags", + "//java/com/google/android/libraries/mobiledatadownload/file", + "//java/com/google/android/libraries/mobiledatadownload/file/backends:file", + "//java/com/google/android/libraries/mobiledatadownload/file/common/testing", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogSampler", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LoggingStateStore", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:SharedPreferencesLoggingState", + "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:FakeTimeSource", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:TestFlags", + "//proto:logs_java_proto_lite", + "@androidx_test", + "@com_google_guava_guava", + "@truth", + ], +) + +mdd_local_test( + name = "DownloadStateLoggerTest", + srcs = ["DownloadStateLoggerTest.java"], + test_class = "com.google.android.libraries.mobiledatadownload.internal.logging.DownloadStateLoggerTest", + deps = [ + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:DownloadStateLogger", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging/testing:FakeEventLogger", + "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", + "//third_party/java/junit", + "@androidx_test", + "@com_google_guava_guava", + "@robolectric", + "@truth", + ], +) diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/DownloadStateLoggerTest.java b/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/DownloadStateLoggerTest.java new file mode 100644 index 0000000..dc37521 --- /dev/null +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/DownloadStateLoggerTest.java @@ -0,0 +1,157 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.internal.logging; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.mobiledatadownload.internal.MetadataProto.DataFile; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupBookkeeping; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; +import com.google.android.libraries.mobiledatadownload.internal.logging.DownloadStateLogger.Operation; +import com.google.android.libraries.mobiledatadownload.internal.logging.testing.FakeEventLogger; +import com.google.common.collect.ImmutableMap; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; +import java.util.Map; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.ParameterizedRobolectricTestRunner; +import org.robolectric.ParameterizedRobolectricTestRunner.Parameter; +import org.robolectric.ParameterizedRobolectricTestRunner.Parameters; + +@RunWith(ParameterizedRobolectricTestRunner.class) +public final class DownloadStateLoggerTest { + + @Parameter(value = 0) + public Operation operation; + + @Parameter(value = 1) + public Map<String, MddClientEvent.Code> expectedCodeMap; + + @Parameters(name = "{index}: operation = {0}, expectedCodeMap = {1}") + public static Object[][] parameters() { + return new Object[][] { + { + Operation.DOWNLOAD, + ImmutableMap.builder() + .put("started", MddClientEvent.Code.EVENT_CODE_UNSPECIFIED) + .put("pending", MddClientEvent.Code.EVENT_CODE_UNSPECIFIED) + .put("failed", MddClientEvent.Code.EVENT_CODE_UNSPECIFIED) + .put("complete", MddClientEvent.Code.EVENT_CODE_UNSPECIFIED) + .buildOrThrow(), + }, + { + Operation.IMPORT, + ImmutableMap.builder() + .put("started", MddClientEvent.Code.EVENT_CODE_UNSPECIFIED) + .put("pending", MddClientEvent.Code.EVENT_CODE_UNSPECIFIED) + .put("failed", MddClientEvent.Code.EVENT_CODE_UNSPECIFIED) + .put("complete", MddClientEvent.Code.EVENT_CODE_UNSPECIFIED) + .buildOrThrow(), + }, + }; + } + + private static final DataFileGroupBookkeeping FILE_GROUP_BOOKKEEPING = + DataFileGroupBookkeeping.newBuilder() + .setGroupNewFilesReceivedTimestamp(100L) + .setGroupDownloadStartedTimestampInMillis(1000L) + .setGroupDownloadedTimestampInMillis(10000L) + .setDownloadStartedCount(5) + .build(); + + private static final DataFileGroupInternal FILE_GROUP = + DataFileGroupInternal.newBuilder() + .setGroupName("test-group") + .setBuildId(100L) + .setVariantId("variant") + .setFileGroupVersionNumber(10) + .addFile(DataFile.getDefaultInstance()) + .setBookkeeping(FILE_GROUP_BOOKKEEPING) + .build(); + + private static final DataDownloadFileGroupStats EXPECTED_FILE_GROUP_STATS = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(FILE_GROUP.getGroupName()) + .setFileGroupVersionNumber(FILE_GROUP.getFileGroupVersionNumber()) + .setBuildId(FILE_GROUP.getBuildId()) + .setVariantId(FILE_GROUP.getVariantId()) + .setOwnerPackage("") + .setFileCount(1) + .build(); + + private static final Void EXPECTED_DOWNLOAD_LATENCY = null; + + private final FakeEventLogger fakeEventLogger = new FakeEventLogger(); + + private DownloadStateLogger downloadStateLogger; + + @Before + public void setUp() { + downloadStateLogger = loggerForOperation(operation); + } + + @Test + public void logStarted_logsExpectedCode() throws Exception { + downloadStateLogger.logStarted(FILE_GROUP); + + assertExpectedCodeIsLogged(expectedCodeMap.get("started")); + } + + @Test + public void logPending_logsExpectedCode() throws Exception { + downloadStateLogger.logPending(FILE_GROUP); + + assertExpectedCodeIsLogged(expectedCodeMap.get("pending")); + } + + @Test + public void logFailed_logsExpectedCode() throws Exception { + downloadStateLogger.logFailed(FILE_GROUP); + + assertExpectedCodeIsLogged(expectedCodeMap.get("failed")); + } + + @Test + public void logComplete_logsExpectedCode() throws Exception { + downloadStateLogger.logComplete(FILE_GROUP); + + assertExpectedCodeIsLogged(expectedCodeMap.get("complete")); + + if (operation == Operation.DOWNLOAD) { + assertThat(fakeEventLogger.getLoggedLatencies()).hasSize(1); + assertThat(fakeEventLogger.getLoggedLatencies()).containsKey(EXPECTED_FILE_GROUP_STATS); + assertThat(fakeEventLogger.getLoggedLatencies().values()).contains(EXPECTED_DOWNLOAD_LATENCY); + } else { + assertThat(fakeEventLogger.getLoggedLatencies()).isEmpty(); + } + } + + private DownloadStateLogger loggerForOperation(Operation operation) { + switch (operation) { + case DOWNLOAD: + return DownloadStateLogger.forDownload(fakeEventLogger); + case IMPORT: + return DownloadStateLogger.forImport(fakeEventLogger); + } + throw new AssertionError(); + } + + private void assertExpectedCodeIsLogged(MddClientEvent.Code code) { + assertThat(fakeEventLogger.getLoggedCodes()).contains(code); + } +} diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/FileGroupStatsLoggerTest.java b/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/FileGroupStatsLoggerTest.java new file mode 100644 index 0000000..0012002 --- /dev/null +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/FileGroupStatsLoggerTest.java @@ -0,0 +1,343 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.internal.logging; + +import static com.google.common.truth.Truth.assertThat; +import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.mobiledatadownload.internal.MetadataProto.DataFile; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupBookkeeping; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; +import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; +import com.google.android.libraries.mobiledatadownload.internal.FileGroupManager; +import com.google.android.libraries.mobiledatadownload.internal.FileGroupManager.GroupDownloadStatus; +import com.google.android.libraries.mobiledatadownload.internal.FileGroupsMetadata; +import com.google.android.libraries.mobiledatadownload.internal.MddTestUtil; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup; +import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupUtil; +import com.google.common.util.concurrent.AsyncCallable; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.mobiledatadownload.LogEnumsProto.MddFileGroupDownloadStatus; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; +import com.google.mobiledatadownload.LogProto.MddFileGroupStatus; +import java.util.ArrayList; +import java.util.List; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.robolectric.RobolectricTestRunner; + +@RunWith(RobolectricTestRunner.class) +public class FileGroupStatsLoggerTest { + + private static final String TEST_GROUP = "test-group"; + private static final String TEST_GROUP_2 = "test-group-2"; + + private static final String TEST_PACKAGE = "test-package"; + + // This one has account + private static final GroupKey TEST_KEY = + GroupKey.newBuilder() + .setGroupName(TEST_GROUP) + .setOwnerPackage(TEST_PACKAGE) + .setAccount("some_account") + .build(); + + // This one does not have account + private static final GroupKey TEST_KEY_2 = + GroupKey.newBuilder().setGroupName(TEST_GROUP_2).setOwnerPackage(TEST_PACKAGE).build(); + + @Mock FileGroupManager mockFileGroupManager; + @Mock FileGroupsMetadata mockFileGroupsMetadata; + @Mock EventLogger mockEventLogger; + + private FileGroupStatsLogger fileGroupStatsLogger; + + @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + + @Captor + ArgumentCaptor<AsyncCallable<List<EventLogger.FileGroupStatusWithDetails>>> + fileGroupStatusAndDetailsListCaptor; + + @Before + public void setUp() throws Exception { + + fileGroupStatsLogger = + new FileGroupStatsLogger( + mockFileGroupManager, + mockFileGroupsMetadata, + mockEventLogger, + MoreExecutors.directExecutor()); + } + + @Test + public void fileGroupStatsLogging() throws Exception { + int daysSinceLastLog = 10; + + List<GroupKeyAndGroup> groups = new ArrayList<>(); + + // Add a downloaded group with version number 10. + DataFileGroupInternal fileGroupDownloaded = + MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 2).toBuilder() + .setFileGroupVersionNumber(10) + .setBuildId(10) + .setVariantId("test-variant") + .build(); + fileGroupDownloaded = + FileGroupUtil.setGroupNewFilesReceivedTimestamp(fileGroupDownloaded, 5000); + fileGroupDownloaded = FileGroupUtil.setDownloadedTimestampInMillis(fileGroupDownloaded, 10000); + + groups.add( + GroupKeyAndGroup.create( + TEST_KEY.toBuilder().setDownloaded(true).build(), fileGroupDownloaded)); + + // Add a pending download group for the same group name with version number 11. + DataFileGroupInternal fileGroupPending = + MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 3).toBuilder() + .setFileGroupVersionNumber(11) + .setStaleLifetimeSecs(0) + .setExpirationDateSecs(0) + .setBookkeeping(DataFileGroupBookkeeping.newBuilder().setStaleExpirationDate(0).build()) + .setBuildId(11) + .setVariantId("test-variant") + .build(); + fileGroupPending = FileGroupUtil.setGroupNewFilesReceivedTimestamp(fileGroupPending, 15000); + groups.add(GroupKeyAndGroup.create(TEST_KEY, fileGroupPending)); + when(mockFileGroupManager.getFileGroupDownloadStatus(fileGroupPending)) + .thenReturn(Futures.immediateFuture(GroupDownloadStatus.PENDING)); + + // Add a failed group to metadata with version 5. + DataFileGroupInternal fileGroupFailed = + MddTestUtil.createDataFileGroupInternal(TEST_GROUP_2, 3).toBuilder() + .setFileGroupVersionNumber(5) + .setStaleLifetimeSecs(0) + .setExpirationDateSecs(0) + .setBookkeeping(DataFileGroupBookkeeping.newBuilder().setStaleExpirationDate(0).build()) + .build(); + fileGroupFailed = FileGroupUtil.setGroupNewFilesReceivedTimestamp(fileGroupFailed, 12000); + groups.add(GroupKeyAndGroup.create(TEST_KEY_2, fileGroupFailed)); + when(mockFileGroupManager.getFileGroupDownloadStatus(fileGroupFailed)) + .thenReturn(Futures.immediateFuture(GroupDownloadStatus.FAILED)); + + when(mockFileGroupsMetadata.getAllFreshGroups()).thenReturn(Futures.immediateFuture(groups)); + + when(mockEventLogger.logMddFileGroupStats(any())).thenReturn(Futures.immediateVoidFuture()); + fileGroupStatsLogger.log(daysSinceLastLog).get(); + + verify(mockEventLogger, times(1)) + .logMddFileGroupStats(fileGroupStatusAndDetailsListCaptor.capture()); + + List<EventLogger.FileGroupStatusWithDetails> allFileGroupStatusAndDetailsList = + fileGroupStatusAndDetailsListCaptor.getValue().call().get(); + MddFileGroupStatus status1 = allFileGroupStatusAndDetailsList.get(0).fileGroupStatus(); + MddFileGroupStatus status2 = allFileGroupStatusAndDetailsList.get(1).fileGroupStatus(); + MddFileGroupStatus status3 = allFileGroupStatusAndDetailsList.get(2).fileGroupStatus(); + + DataDownloadFileGroupStats details1 = + allFileGroupStatusAndDetailsList.get(0).fileGroupDetails(); + DataDownloadFileGroupStats details2 = + allFileGroupStatusAndDetailsList.get(1).fileGroupDetails(); + DataDownloadFileGroupStats details3 = + allFileGroupStatusAndDetailsList.get(2).fileGroupDetails(); + + // Check that the downloaded group status is logged. + assertThat(details1.getFileGroupName()).isEqualTo(TEST_GROUP); + assertThat(details1.getOwnerPackage()).isEqualTo(TEST_PACKAGE); + assertThat(details1.getFileGroupVersionNumber()).isEqualTo(10); + assertThat(details1.getBuildId()).isEqualTo(10); + assertThat(details1.getVariantId()).isEqualTo("test-variant"); + assertThat(details1.getFileCount()).isEqualTo(2); + assertThat(details1.getInlineFileCount()).isEqualTo(0); + assertTrue(details1.getHasAccount()); + assertThat(status1.getFileGroupDownloadStatus()) + .isEqualTo(MddFileGroupDownloadStatus.Code.COMPLETE); + assertThat(status1.getGroupAddedTimestampInSeconds()).isEqualTo(5); + assertThat(status1.getGroupDownloadedTimestampInSeconds()).isEqualTo(10); + assertThat(status1.getDaysSinceLastLog()).isEqualTo(daysSinceLastLog); + + // Check that the pending group status is logged. + assertThat(details2.getFileGroupName()).isEqualTo(TEST_GROUP); + assertThat(details2.getFileGroupVersionNumber()).isEqualTo(11); + assertThat(details2.getBuildId()).isEqualTo(11); + assertThat(details2.getVariantId()).isEqualTo("test-variant"); + assertThat(details2.getOwnerPackage()).isEqualTo(TEST_PACKAGE); + assertThat(details2.getFileCount()).isEqualTo(3); + assertThat(details2.getInlineFileCount()).isEqualTo(0); + assertTrue(details2.getHasAccount()); + assertThat(status2.getFileGroupDownloadStatus()) + .isEqualTo(MddFileGroupDownloadStatus.Code.PENDING); + assertThat(status2.getGroupAddedTimestampInSeconds()).isEqualTo(15); + assertThat(status2.getGroupDownloadedTimestampInSeconds()).isEqualTo(-1); + assertThat(status2.getDaysSinceLastLog()).isEqualTo(daysSinceLastLog); + + // Check that the failed group status is logged. + assertThat(details3.getFileGroupName()).isEqualTo(TEST_GROUP_2); + assertThat(details3.getFileGroupVersionNumber()).isEqualTo(5); + assertThat(details3.getOwnerPackage()).isEqualTo(TEST_PACKAGE); + assertThat(details3.getFileCount()).isEqualTo(3); + assertThat(details3.getInlineFileCount()).isEqualTo(0); + assertFalse(details3.getHasAccount()); + assertThat(status3.getFileGroupDownloadStatus()) + .isEqualTo(MddFileGroupDownloadStatus.Code.FAILED); + assertThat(status3.getGroupAddedTimestampInSeconds()).isEqualTo(12); + assertThat(status3.getGroupDownloadedTimestampInSeconds()).isEqualTo(-1); + assertThat(status3.getDaysSinceLastLog()).isEqualTo(daysSinceLastLog); + } + + @Test + public void fileGroupStatsLogging_withInlineFiles() throws Exception { + int daysSinceLastLog = 10; + + List<GroupKeyAndGroup> groups = new ArrayList<>(); + + DataFile inlineFile1 = + DataFile.newBuilder() + .setFileId("inline-file") + .setUrlToDownload("inlinefile:sha1:checksum") + .setChecksum("checksum") + .setByteSize(10) + .build(); + DataFile inlineFile2 = + DataFile.newBuilder() + .setFileId("inline-file-2") + .setUrlToDownload("inlinefile:sha1:checksum2") + .setChecksum("checksum2") + .setByteSize(11) + .build(); + + // Add a downloaded group with version number 10 and inline file + DataFileGroupInternal fileGroupDownloaded = + MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 2).toBuilder() + .setFileGroupVersionNumber(10) + .setBuildId(10) + .setVariantId("test-variant") + .addFile(inlineFile1) + .addFile(inlineFile2) + .build(); + fileGroupDownloaded = + FileGroupUtil.setGroupNewFilesReceivedTimestamp(fileGroupDownloaded, 5000); + fileGroupDownloaded = FileGroupUtil.setDownloadedTimestampInMillis(fileGroupDownloaded, 10000); + + groups.add( + GroupKeyAndGroup.create( + TEST_KEY.toBuilder().setDownloaded(true).build(), fileGroupDownloaded)); + + // Add a pending download group for the same group name with version number 11 and inline file. + DataFileGroupInternal fileGroupPending = + MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 3).toBuilder() + .setFileGroupVersionNumber(11) + .setStaleLifetimeSecs(0) + .setExpirationDateSecs(0) + .setBookkeeping(DataFileGroupBookkeeping.newBuilder().setStaleExpirationDate(0).build()) + .setBuildId(11) + .setVariantId("test-variant") + .addFile(inlineFile1) + .build(); + fileGroupPending = FileGroupUtil.setGroupNewFilesReceivedTimestamp(fileGroupPending, 15000); + groups.add(GroupKeyAndGroup.create(TEST_KEY, fileGroupPending)); + when(mockFileGroupManager.getFileGroupDownloadStatus(fileGroupPending)) + .thenReturn(Futures.immediateFuture(GroupDownloadStatus.PENDING)); + + // Add a failed group to metadata with version 5 with no inline files. + DataFileGroupInternal fileGroupFailed = + MddTestUtil.createDataFileGroupInternal(TEST_GROUP_2, 3).toBuilder() + .setFileGroupVersionNumber(5) + .setStaleLifetimeSecs(0) + .setExpirationDateSecs(0) + .setBookkeeping(DataFileGroupBookkeeping.newBuilder().setStaleExpirationDate(0).build()) + .build(); + fileGroupFailed = FileGroupUtil.setGroupNewFilesReceivedTimestamp(fileGroupFailed, 12000); + groups.add(GroupKeyAndGroup.create(TEST_KEY_2, fileGroupFailed)); + when(mockFileGroupManager.getFileGroupDownloadStatus(fileGroupFailed)) + .thenReturn(Futures.immediateFuture(GroupDownloadStatus.FAILED)); + + when(mockFileGroupsMetadata.getAllFreshGroups()).thenReturn(Futures.immediateFuture(groups)); + when(mockEventLogger.logMddFileGroupStats(any())).thenReturn(Futures.immediateVoidFuture()); + + fileGroupStatsLogger.log(daysSinceLastLog).get(); + + verify(mockEventLogger, times(1)) + .logMddFileGroupStats(fileGroupStatusAndDetailsListCaptor.capture()); + + List<EventLogger.FileGroupStatusWithDetails> allFileGroupStatusAndDetailsList = + fileGroupStatusAndDetailsListCaptor.getValue().call().get(); + MddFileGroupStatus status1 = allFileGroupStatusAndDetailsList.get(0).fileGroupStatus(); + MddFileGroupStatus status2 = allFileGroupStatusAndDetailsList.get(1).fileGroupStatus(); + MddFileGroupStatus status3 = allFileGroupStatusAndDetailsList.get(2).fileGroupStatus(); + + DataDownloadFileGroupStats details1 = + allFileGroupStatusAndDetailsList.get(0).fileGroupDetails(); + DataDownloadFileGroupStats details2 = + allFileGroupStatusAndDetailsList.get(1).fileGroupDetails(); + DataDownloadFileGroupStats details3 = + allFileGroupStatusAndDetailsList.get(2).fileGroupDetails(); + + // Check that the downloaded group status is logged. + assertThat(details1.getFileGroupName()).isEqualTo(TEST_GROUP); + assertThat(details1.getOwnerPackage()).isEqualTo(TEST_PACKAGE); + assertThat(details1.getFileGroupVersionNumber()).isEqualTo(10); + assertThat(details1.getBuildId()).isEqualTo(10); + assertThat(details1.getVariantId()).isEqualTo("test-variant"); + assertThat(details1.getFileCount()).isEqualTo(4); + assertThat(details1.getInlineFileCount()).isEqualTo(2); + assertTrue(details1.getHasAccount()); + assertThat(status1.getFileGroupDownloadStatus()) + .isEqualTo(MddFileGroupDownloadStatus.Code.COMPLETE); + assertThat(status1.getGroupAddedTimestampInSeconds()).isEqualTo(5); + assertThat(status1.getGroupDownloadedTimestampInSeconds()).isEqualTo(10); + assertThat(status1.getDaysSinceLastLog()).isEqualTo(daysSinceLastLog); + + // Check that the pending group status is logged. + assertThat(details2.getFileGroupName()).isEqualTo(TEST_GROUP); + assertThat(details2.getFileGroupVersionNumber()).isEqualTo(11); + assertThat(details2.getBuildId()).isEqualTo(11); + assertThat(details2.getVariantId()).isEqualTo("test-variant"); + assertThat(details2.getOwnerPackage()).isEqualTo(TEST_PACKAGE); + assertThat(details2.getFileCount()).isEqualTo(4); + assertThat(details2.getInlineFileCount()).isEqualTo(1); + assertTrue(details2.getHasAccount()); + assertThat(status2.getFileGroupDownloadStatus()) + .isEqualTo(MddFileGroupDownloadStatus.Code.PENDING); + assertThat(status2.getGroupAddedTimestampInSeconds()).isEqualTo(15); + assertThat(status2.getGroupDownloadedTimestampInSeconds()).isEqualTo(-1); + assertThat(status2.getDaysSinceLastLog()).isEqualTo(daysSinceLastLog); + + // Check that the failed group status is logged. + assertThat(details3.getFileGroupName()).isEqualTo(TEST_GROUP_2); + assertThat(details3.getFileGroupVersionNumber()).isEqualTo(5); + assertThat(details3.getOwnerPackage()).isEqualTo(TEST_PACKAGE); + assertThat(details3.getFileCount()).isEqualTo(3); + assertThat(details3.getInlineFileCount()).isEqualTo(0); + assertFalse(details3.getHasAccount()); + assertThat(status3.getFileGroupDownloadStatus()) + .isEqualTo(MddFileGroupDownloadStatus.Code.FAILED); + assertThat(status3.getGroupAddedTimestampInSeconds()).isEqualTo(12); + assertThat(status3.getGroupDownloadedTimestampInSeconds()).isEqualTo(-1); + assertThat(status3.getDaysSinceLastLog()).isEqualTo(daysSinceLastLog); + } +} diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/LogSamplerTest.java b/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/LogSamplerTest.java new file mode 100644 index 0000000..a06d8ea --- /dev/null +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/LogSamplerTest.java @@ -0,0 +1,232 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.internal.logging; + +import static com.google.android.libraries.mobiledatadownload.internal.logging.SharedPreferencesLoggingState.SALT_KEY; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assume.assumeTrue; + +import android.content.Context; +import android.content.SharedPreferences; +import androidx.test.core.app.ApplicationProvider; +import com.google.android.libraries.mobiledatadownload.Flags; +import com.google.android.libraries.mobiledatadownload.file.common.testing.TemporaryUri; +import com.google.android.libraries.mobiledatadownload.testing.FakeTimeSource; +import com.google.common.base.Optional; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.mobiledatadownload.LogProto.StableSamplingInfo; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.concurrent.Executors; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.ParameterizedRobolectricTestRunner; +import org.robolectric.ParameterizedRobolectricTestRunner.Parameter; +import org.robolectric.ParameterizedRobolectricTestRunner.Parameters; + +@RunWith(ParameterizedRobolectricTestRunner.class) +public final class LogSamplerTest { + @Parameter(value = 0) + public boolean stableLoggingEnabled; + + @Parameters(name = "stableLoggingEnabled = {0}") + public static List<Boolean> parameters() { + return Arrays.asList(true, false); + } + + private LoggingStateStore loggingStateStore; + private SharedPreferences loggingStateSharedPrefs; + private static final ListeningExecutorService executorService = + MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); + private LogSampler logSampler; + + @Rule public final TemporaryUri tmpUri = new TemporaryUri(); + + private static final FakeTimeSource timeSource = new FakeTimeSource(); + private Context context; + + // Seed for first long + private static final int LOGS_AT_1_PERCENT_SEED = 750; // -5772485602628857500 + + private static final int ONE_PERCENT_SAMPLE_INTERVAL = 100; + private static final int TEN_PERCENT_SAMPLE_INTERVAL = 10; + private static final int ONE_HUNDRED_PERCENT_SAMPLE_INTERVAL = 1; + private static final int NEVER_SAMPLE_INTERVAL = 0; + + @Before + public void setUp() throws Exception { + context = ApplicationProvider.getApplicationContext(); + + loggingStateSharedPrefs = context.getSharedPreferences("loggingStateSharedPrefs", 0); + + loggingStateStore = + SharedPreferencesLoggingState.create( + () -> loggingStateSharedPrefs, timeSource, executorService, new Random()); + + logSampler = constructLogSampler(0); + } + + @Test + public void shouldLog_withInvalidSamplingRate_returnsAbsent() throws Exception { + int invalidSamplingRate = -1; + Optional<StableSamplingInfo> samplingInfo = + logSampler.shouldLog(invalidSamplingRate, Optional.of(loggingStateStore)).get(); + + assertThat(samplingInfo).isAbsent(); + } + + @Test + public void shouldLog_with0SamplingRate_returnsAbsent() throws Exception { + Optional<StableSamplingInfo> samplingInfo = + logSampler.shouldLog(NEVER_SAMPLE_INTERVAL, Optional.of(loggingStateStore)).get(); + + assertThat(samplingInfo).isAbsent(); + } + + @Test + public void shouldLog_stable_with1PercentGroup_logsAt1Percent() throws Exception { + assumeTrue(stableLoggingEnabled); + setStableSamplingRandomNumber(100); // 100 % 100 = 0 + + Optional<StableSamplingInfo> samplingInfo = + logSampler.shouldLog(ONE_PERCENT_SAMPLE_INTERVAL, Optional.of(loggingStateStore)).get(); + + assertThat(samplingInfo).isPresent(); + assertThat(samplingInfo.get().getStableSamplingUsed()).isTrue(); + assertThat(samplingInfo.get().getPartOfAlwaysLoggingGroup()).isTrue(); + assertThat(samplingInfo.get().getInvalidSamplingRateUsed()).isFalse(); + } + + @Test + public void shouldLog_stable_with1PercentGroup_logsAt10Percent() throws Exception { + assumeTrue(stableLoggingEnabled); + setStableSamplingRandomNumber(100); // 100 % 100 = 0 + + Optional<StableSamplingInfo> samplingInfo = + logSampler.shouldLog(TEN_PERCENT_SAMPLE_INTERVAL, Optional.of(loggingStateStore)).get(); + + assertThat(samplingInfo).isPresent(); + assertThat(samplingInfo.get().getStableSamplingUsed()).isTrue(); + assertThat(samplingInfo.get().getPartOfAlwaysLoggingGroup()).isTrue(); + assertThat(samplingInfo.get().getInvalidSamplingRateUsed()).isFalse(); + } + + @Test + public void shouldLog_stable_with10PercentGroup_doesntLogAt1Percent() throws Exception { + assumeTrue(stableLoggingEnabled); + setStableSamplingRandomNumber(10); // 10 % 100 = 10 + + Optional<StableSamplingInfo> samplingInfo = + logSampler.shouldLog(ONE_PERCENT_SAMPLE_INTERVAL, Optional.of(loggingStateStore)).get(); + + assertThat(samplingInfo).isAbsent(); + } + + @Test + public void shouldLog_stable_with10PercentGroup_logsAt10Percent() throws Exception { + assumeTrue(stableLoggingEnabled); + setStableSamplingRandomNumber(10); // 10 % 100 = 10 + + Optional<StableSamplingInfo> samplingInfo = + logSampler.shouldLog(TEN_PERCENT_SAMPLE_INTERVAL, Optional.of(loggingStateStore)).get(); + + assertThat(samplingInfo).isPresent(); + assertThat(samplingInfo.get().getStableSamplingUsed()).isTrue(); + assertThat(samplingInfo.get().getPartOfAlwaysLoggingGroup()).isFalse(); + assertThat(samplingInfo.get().getInvalidSamplingRateUsed()).isFalse(); + } + + @Test + public void shouldLog_stable_withIncompatibleSamplingRate_isMarkedAsIncompatible() + throws Exception { + assumeTrue(stableLoggingEnabled); + setStableSamplingRandomNumber(77); + + Optional<StableSamplingInfo> samplingInfo = + logSampler.shouldLog(77, Optional.of(loggingStateStore)).get(); + + assertThat(samplingInfo).isPresent(); + assertThat(samplingInfo.get().getStableSamplingUsed()).isTrue(); + assertThat(samplingInfo.get().getInvalidSamplingRateUsed()).isTrue(); + assertThat(samplingInfo.get().getPartOfAlwaysLoggingGroup()).isFalse(); + } + + @Test + public void shouldLog_with100Percent_logsAt100Percent() throws Exception { + Optional<StableSamplingInfo> samplingInfo1 = + logSampler + .shouldLog(ONE_HUNDRED_PERCENT_SAMPLE_INTERVAL, Optional.of(loggingStateStore)) + .get(); + Optional<StableSamplingInfo> samplingInfo2 = + logSampler + .shouldLog(ONE_HUNDRED_PERCENT_SAMPLE_INTERVAL, Optional.of(loggingStateStore)) + .get(); + + assertThat(samplingInfo1).isPresent(); + assertThat(samplingInfo2).isPresent(); + assertThat(samplingInfo1.get().getStableSamplingUsed()).isEqualTo(stableLoggingEnabled); + assertThat(samplingInfo2.get().getStableSamplingUsed()).isEqualTo(stableLoggingEnabled); + } + + @Test + public void shouldLog_event_changesPerEvent() throws Exception { + assumeTrue(!stableLoggingEnabled); + + LogSampler logSampler = constructLogSampler(LOGS_AT_1_PERCENT_SEED); + checkState( + logSampler + .shouldLog(ONE_PERCENT_SAMPLE_INTERVAL, Optional.of(loggingStateStore)) + .get() + .isPresent()); + + assertThat( + logSampler.shouldLog(ONE_PERCENT_SAMPLE_INTERVAL, Optional.of(loggingStateStore)).get()) + .isAbsent(); + } + + @Test + public void shouldLog_stable_withoutLoggingStateStore_usesPerEvent() throws Exception { + assumeTrue(stableLoggingEnabled); + + Optional<StableSamplingInfo> stableSamplingInfo = + logSampler.shouldLog(ONE_HUNDRED_PERCENT_SAMPLE_INTERVAL, Optional.absent()).get(); + + assertThat(stableSamplingInfo).isPresent(); + assertThat(stableSamplingInfo.get().getStableSamplingUsed()).isFalse(); + } + + private LogSampler constructLogSampler(int seed) { + return new LogSampler( + new Flags() { + @Override + public boolean enableRngBasedDeviceStableSampling() { + return stableLoggingEnabled; + } + }, + new Random(seed)); + } + + private void setStableSamplingRandomNumber(int randomNumber) throws Exception { + SharedPreferences.Editor editor = loggingStateSharedPrefs.edit(); + editor.putLong(SALT_KEY, randomNumber); + assumeTrue(editor.commit()); + } +} diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/LoggingStateStoreTest.java b/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/LoggingStateStoreTest.java new file mode 100644 index 0000000..98418ad --- /dev/null +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/LoggingStateStoreTest.java @@ -0,0 +1,418 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.internal.logging; + +import static com.google.android.libraries.mobiledatadownload.internal.MddConstants.SPLIT_CHAR; +import static com.google.common.truth.Truth.assertThat; +import static java.util.concurrent.TimeUnit.DAYS; +import static java.util.concurrent.TimeUnit.HOURS; +import static java.util.concurrent.TimeUnit.MINUTES; + +import android.content.Context; +import android.content.SharedPreferences; +import android.net.Uri; +import androidx.test.core.app.ApplicationProvider; +import com.google.mobiledatadownload.internal.MetadataProto.FileGroupLoggingState; +import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; +import com.google.mobiledatadownload.internal.MetadataProto.SamplingInfo; +import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; +import com.google.android.libraries.mobiledatadownload.file.common.testing.FakeFileBackend; +import com.google.android.libraries.mobiledatadownload.file.common.testing.TemporaryUri; +import com.google.android.libraries.mobiledatadownload.testing.FakeTimeSource; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.protobuf.util.Timestamps; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.Executors; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.ParameterizedRobolectricTestRunner; +import org.robolectric.ParameterizedRobolectricTestRunner.Parameters; + +@RunWith(ParameterizedRobolectricTestRunner.class) +public final class LoggingStateStoreTest { + + private static final String OWNER_PACKAGE = "owner-package"; + private static final String VARIANT_ID = "variant-id-1"; + + private static final String GROUP_NAME_1 = "group-name-1"; + private static final String GROUP_NAME_2 = "group-name-2"; + + private static final int BUILD_ID_1 = 1; + + private static final int VERSION_NUMBER_1 = 1; + private static final int VERSION_NUMBER_2 = 2; + + private static final String INSTANCE_ID = "instance-id"; + + private static final ListeningExecutorService executorService = + MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); + + private static final long RANDOM_TESTING_SEED = 1234; + // First long that seed "1234" generates: + private static final long RANDOM_FIRST_SEEDED_LONG = -6519408338692630574L; + + @Rule public final TemporaryUri tmpUri = new TemporaryUri(); + + private Uri uri; + private LoggingStateStore loggingStateStore; + private SharedPreferences loggingStateSharedPrefs; + + private FakeTimeSource timeSource; + private FakeFileBackend fakeFileBackend; + + private Context context; + + /* Run the same test suite on two implementations of the same interface. */ + private enum Implementation { + SHARED_PREFERENCES, + } + + @Parameters(name = "implementation={0}") + public static ImmutableList<Object[]> data() { + return ImmutableList.of(new Object[] {Implementation.SHARED_PREFERENCES}); + } + + private final Implementation implUnderTest; + + public LoggingStateStoreTest(Implementation impl) { + this.implUnderTest = impl; + } + + @Before + public void setUp() throws Exception { + context = ApplicationProvider.getApplicationContext(); + + fakeFileBackend = new FakeFileBackend(); + + SynchronousFileStorage fileStorage = new SynchronousFileStorage(Arrays.asList(fakeFileBackend)); + + Uri uriWithoutPb = tmpUri.newUri(); + + uri = uriWithoutPb.buildUpon().path(uriWithoutPb.getPath() + ".pb").build(); + timeSource = new FakeTimeSource(); + + loggingStateSharedPrefs = context.getSharedPreferences("loggingStateSharedPrefs", 0); + + loggingStateStore = createLoggingStateStore(); + } + + @After + public void cleanUp() throws Exception {} + + @Test + public void testGetAndReset_onFirstRun_returnAbsent() throws Exception { + assertThat(loggingStateStore.getAndResetDaysSinceLastMaintenance().get()).isAbsent(); + } + + @Test + public void testGetAndReset_returnsCorrectNumber() throws Exception { + assertThat(loggingStateStore.getAndResetDaysSinceLastMaintenance().get()).isAbsent(); + timeSource.advance(5, DAYS); + assertThat(loggingStateStore.getAndResetDaysSinceLastMaintenance().get()).hasValue(5); + } + + @Test + public void testGetAndReset_onSameDay_returns0() throws Exception { + assertThat(loggingStateStore.getAndResetDaysSinceLastMaintenance().get()).isAbsent(); + timeSource.advance(1, HOURS); + assertThat(loggingStateStore.getAndResetDaysSinceLastMaintenance().get()).hasValue(0); + timeSource.advance(22, HOURS); + assertThat(loggingStateStore.getAndResetDaysSinceLastMaintenance().get()).hasValue(0); + timeSource.advance(59, MINUTES); + assertThat(loggingStateStore.getAndResetDaysSinceLastMaintenance().get()).hasValue(0); + timeSource.advance(1, MINUTES); + assertThat(loggingStateStore.getAndResetDaysSinceLastMaintenance().get()).hasValue(1); + } + + @Test + public void testGetAndReset_resetsForFuturedays() throws Exception { + assertThat(loggingStateStore.getAndResetDaysSinceLastMaintenance().get()).isAbsent(); + + timeSource.advance(1, DAYS); + assertThat(loggingStateStore.getAndResetDaysSinceLastMaintenance().get()).hasValue(1); + timeSource.advance(1, DAYS); + assertThat(loggingStateStore.getAndResetDaysSinceLastMaintenance().get()).hasValue(1); + } + + @Test + public void testGetAndReset_usesUtcTime() throws Exception { + timeSource.set(1623455940000L); // June 11th 11:59 pm + assertThat(loggingStateStore.getAndResetDaysSinceLastMaintenance().get()).isAbsent(); + timeSource.advance(1, MINUTES); // advance to june 12th + assertThat(loggingStateStore.getAndResetDaysSinceLastMaintenance().get()).hasValue(1); + } + + @Test + public void testGetAndReset_returnsNegativeValue_ifGoesBackInTime() throws Exception { + timeSource.set(1623369600000L); // June 11th 2021 12:00 am + assertThat(loggingStateStore.getAndResetDaysSinceLastMaintenance().get()).isAbsent(); + timeSource.set(1623283200000L); // June 10th 2021 12:00 am + assertThat(loggingStateStore.getAndResetDaysSinceLastMaintenance().get()).hasValue(-1); + } + + @Test + public void testStateIsStoredAcrossRestarts() throws Exception { + assertThat(loggingStateStore.getAndResetDaysSinceLastMaintenance().get()).isAbsent(); + timeSource.advance(20, DAYS); + loggingStateStore = createLoggingStateStore(); + + assertThat(loggingStateStore.getAndResetDaysSinceLastMaintenance().get()).hasValue(20); + } + + @Test + public void testIncrementDataUsage() throws Exception { + FileGroupLoggingState group1FileGroupLoggingState = + FileGroupLoggingState.newBuilder() + .setGroupKey( + GroupKey.newBuilder() + .setGroupName(GROUP_NAME_1) + .setOwnerPackage(OWNER_PACKAGE) + .setVariantId(VARIANT_ID) + .build()) + .setFileGroupVersionNumber(VERSION_NUMBER_1) + .setBuildId(BUILD_ID_1) + .setCellularUsage(123) + .setWifiUsage(456) + .build(); + + loggingStateStore.incrementDataUsage(group1FileGroupLoggingState).get(); + + assertThat(loggingStateStore.getAndResetAllDataUsage().get()) + .containsExactly(group1FileGroupLoggingState); + } + + @Test + public void testIncrementDataUsage_mergesDuplicateEntries() throws Exception { + FileGroupLoggingState group1FileGroupLoggingState = + FileGroupLoggingState.newBuilder() + .setGroupKey( + GroupKey.newBuilder() + .setGroupName(GROUP_NAME_1) + .setOwnerPackage(OWNER_PACKAGE) + .setVariantId(VARIANT_ID) + .build()) + .setFileGroupVersionNumber(VERSION_NUMBER_1) + .setBuildId(BUILD_ID_1) + .setCellularUsage(123) + .setWifiUsage(456) + .build(); + + FileGroupLoggingState withDifferentIncrements = + group1FileGroupLoggingState.toBuilder().setCellularUsage(5).setWifiUsage(10).build(); + + // Increment with build 1 twice + loggingStateStore.incrementDataUsage(group1FileGroupLoggingState).get(); + loggingStateStore.incrementDataUsage(group1FileGroupLoggingState).get(); + + // Increment with varying group name, owner package, variant id, version number, build id. None + // of them should be joined with the unmodified group. + loggingStateStore + .incrementDataUsage(withDifferentIncrements.toBuilder().setBuildId(789).build()) + .get(); + + loggingStateStore + .incrementDataUsage( + withDifferentIncrements.toBuilder().setFileGroupVersionNumber(789).build()) + .get(); + + loggingStateStore + .incrementDataUsage( + withDifferentIncrements.toBuilder() + .setGroupKey( + withDifferentIncrements.getGroupKey().toBuilder() + .setOwnerPackage("someotherpackage")) + .build()) + .get(); + + loggingStateStore + .incrementDataUsage( + withDifferentIncrements.toBuilder() + .setGroupKey( + withDifferentIncrements.getGroupKey().toBuilder().setGroupName("someothername")) + .build()) + .get(); + + loggingStateStore + .incrementDataUsage( + withDifferentIncrements.toBuilder() + .setGroupKey( + withDifferentIncrements.getGroupKey().toBuilder() + .setVariantId("someothervariant")) + .build()) + .get(); + + List<FileGroupLoggingState> allDataUsage = loggingStateStore.getAndResetAllDataUsage().get(); + + assertThat(allDataUsage) + .contains( + group1FileGroupLoggingState.toBuilder() + .setCellularUsage(group1FileGroupLoggingState.getCellularUsage() * 2) + .setWifiUsage(group1FileGroupLoggingState.getWifiUsage() * 2) + .build()); + + assertThat(allDataUsage).hasSize(6); + } + + @Test + public void testGetAndResetDataUsage_resetsAllDataUsage() throws Exception { + FileGroupLoggingState group1FileGroupLoggingState = + FileGroupLoggingState.newBuilder() + .setGroupKey( + GroupKey.newBuilder() + .setGroupName(GROUP_NAME_1) + .setOwnerPackage(OWNER_PACKAGE) + .setVariantId(VARIANT_ID) + .build()) + .setFileGroupVersionNumber(VERSION_NUMBER_1) + .setBuildId(BUILD_ID_1) + .setCellularUsage(123) + .setWifiUsage(456) + .build(); + + loggingStateStore.incrementDataUsage(group1FileGroupLoggingState).get(); + + assertThat(loggingStateStore.getAndResetAllDataUsage().get()) + .containsExactly(group1FileGroupLoggingState); + + assertThat(loggingStateStore.getAndResetAllDataUsage().get()).isEmpty(); + } + + @Test + public void testClear_clearsAllState() throws Exception { + assertThat(loggingStateStore.getAndResetDaysSinceLastMaintenance().get()).isAbsent(); + timeSource.advance(20, DAYS); + + FileGroupLoggingState group1FileGroupLoggingState = + FileGroupLoggingState.newBuilder() + .setGroupKey( + GroupKey.newBuilder() + .setGroupName(GROUP_NAME_1) + .setOwnerPackage(OWNER_PACKAGE) + .setVariantId(VARIANT_ID) + .build()) + .setFileGroupVersionNumber(VERSION_NUMBER_1) + .setBuildId(BUILD_ID_1) + .setCellularUsage(123) + .setWifiUsage(456) + .build(); + + loggingStateStore.incrementDataUsage(group1FileGroupLoggingState).get(); + + loggingStateStore.clear().get(); + + assertThat(loggingStateStore.getAndResetDaysSinceLastMaintenance().get()).isAbsent(); + assertThat(loggingStateStore.getAndResetAllDataUsage().get()).isEmpty(); + } + + @Test + public void testGetSamplingInfo_returnsPopulatedSamplingInfo() throws Exception { + long timeMillis = 1234567890L; + timeSource.set(timeMillis); + + SamplingInfo samplingInfo = loggingStateStore.getStableSamplingInfo().get(); + + assertThat(samplingInfo.getStableLogSamplingSalt()).isEqualTo(RANDOM_FIRST_SEEDED_LONG); + assertThat(samplingInfo.getLogSamplingSaltSetTimestamp()) + .isEqualTo(Timestamps.fromMillis(timeMillis)); + } + + @Test + public void testGetSamplingInfo_seedsWithProvidedRngAndTimestamp() throws Exception { + timeSource.set(12345L); + loggingStateStore.getAndResetDaysSinceLastMaintenance().get(); // Should not be affected + + long timeMillis = 1234567890L; + timeSource.set(timeMillis); + + SamplingInfo samplingInfo = loggingStateStore.getStableSamplingInfo().get(); + + assertThat(samplingInfo) + .isEqualTo( + SamplingInfo.newBuilder() + .setStableLogSamplingSalt(RANDOM_FIRST_SEEDED_LONG) + .setLogSamplingSaltSetTimestamp(Timestamps.fromMillis(timeMillis)) + .build()); + // 1234567890 - 12345 millis = 14 days + assertThat(loggingStateStore.getAndResetDaysSinceLastMaintenance().get()).hasValue(14); + } + + @Test + public void testGetSamplingInfo_doesNotModifyExistingSamplingData() throws Exception { + timeSource.set(12345L); + LoggingStateStore existingStore = createLoggingStateStore(); + existingStore.getStableSamplingInfo().get(); // Should not be affected + + long timeMillis = 1234567890L; + timeSource.set(timeMillis); + + SamplingInfo samplingInfo = loggingStateStore.getStableSamplingInfo().get(); + + assertThat(samplingInfo.getStableLogSamplingSalt()).isEqualTo(RANDOM_FIRST_SEEDED_LONG); + assertThat(samplingInfo.getLogSamplingSaltSetTimestamp()) + .isEqualTo(Timestamps.fromMillis(12345L)); + } + + private static String getFileGroupKey( + String ownerPackage, String groupName, int versionNumber, String networkType) { + // Format of shared preferences key is: ownerPackage|groupName|versionNumber|networkType, value + // is: long. + return new StringBuilder(ownerPackage) + .append(SPLIT_CHAR) + .append(groupName) + .append(SPLIT_CHAR) + .append(versionNumber) + .append(SPLIT_CHAR) + .append(networkType) + .toString(); + } + + /** + * Adds the preferences from {@code prefsToAdd} to {@code prefs}. Throws an Exception if it fails + * to write to the SharedPreferences (e.g. to IO errors). + */ + private static void addPreferencesOrThrow( + SharedPreferences prefs, ImmutableMap<String, Long> prefsToAdd) { + SharedPreferences.Editor editor = prefs.edit(); + for (Map.Entry<String, Long> entryToWrite : prefsToAdd.entrySet()) { + editor.putLong(entryToWrite.getKey(), entryToWrite.getValue()); + } + + Preconditions.checkState( + editor.commit(), "Unable to write to shared prefs when setting up test."); + } + + private LoggingStateStore createLoggingStateStore() throws Exception { + switch (implUnderTest) { + case SHARED_PREFERENCES: + return SharedPreferencesLoggingState.create( + () -> loggingStateSharedPrefs, + timeSource, + executorService, + new Random(RANDOM_TESTING_SEED)); + } + throw new AssertionError(); // Exhaustive switch + } +} diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/MddEventLoggerTest.java b/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/MddEventLoggerTest.java new file mode 100644 index 0000000..468f7fe --- /dev/null +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/MddEventLoggerTest.java @@ -0,0 +1,272 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.internal.logging; + +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +import android.content.Context; +import androidx.test.core.app.ApplicationProvider; +import com.google.android.libraries.mobiledatadownload.Logger; +import com.google.android.libraries.mobiledatadownload.testing.FakeTimeSource; +import com.google.android.libraries.mobiledatadownload.testing.MddTestDependencies; +import com.google.android.libraries.mobiledatadownload.testing.TestFlags; +import com.google.common.base.Optional; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; +import com.google.mobiledatadownload.LogEnumsProto.MddDownloadResult; +import com.google.mobiledatadownload.LogProto.AndroidClientInfo; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; +import com.google.mobiledatadownload.LogProto.MddDeviceInfo; +import com.google.mobiledatadownload.LogProto.MddDownloadResultLog; +import com.google.mobiledatadownload.LogProto.MddLogData; +import com.google.mobiledatadownload.LogProto.StableSamplingInfo; +import java.security.SecureRandom; +import java.util.Random; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.robolectric.RobolectricTestRunner; + +@RunWith(RobolectricTestRunner.class) +public class MddEventLoggerTest { + + @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + + private static final int SOME_MODULE_VERSION = 42; + private static final int SAMPLING_ALWAYS = 1; + private static final int SAMPLING_NEVER = 0; + + @Mock private Logger mockLogger; + private MddEventLogger mddEventLogger; + + private final Context context = ApplicationProvider.getApplicationContext(); + private final TestFlags flags = new TestFlags(); + + @Before + public void setUp() throws Exception { + mddEventLogger = + new MddEventLogger( + context, + mockLogger, + SOME_MODULE_VERSION, + new LogSampler(flags, new SecureRandom()), + flags); + mddEventLogger.setLoggingStateStore( + MddTestDependencies.LoggingStateStoreImpl.SHARED_PREFERENCES.loggingStateStore( + context, Optional.absent(), new FakeTimeSource(), directExecutor(), new Random(0))); + } + + private MddLogData.Builder newLogDataBuilderWithClientInfo() { + return MddLogData.newBuilder() + .setAndroidClientInfo( + AndroidClientInfo.newBuilder() + .setModuleVersion(SOME_MODULE_VERSION) + .setHostPackageName(context.getPackageName())); + } + + @Test + public void testSampleInterval_zero_none() { + assertFalse(LogUtil.shouldSampleInterval(0)); + } + + @Test + public void testSampleInterval_negative_none() { + assertFalse(LogUtil.shouldSampleInterval(-1)); + } + + @Test + public void testSampleInterval_always() { + assertTrue(LogUtil.shouldSampleInterval(1)); + } + + @Test + public void testLogMddEvents_noLog() { + overrideDefaultSampleInterval(SAMPLING_NEVER); + + mddEventLogger.logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + "fileGroup", + /* fileGroupVersionNumber= */ 0, + /* buildId= */ 0, + /* variantId= */ ""); + verifyNoInteractions(mockLogger); + } + + @Test + public void testLogMddEvents() { + overrideDefaultSampleInterval(SAMPLING_ALWAYS); + mddEventLogger.logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + "fileGroup", + /* fileGroupVersionNumber= */ 1, + /* buildId= */ 123, + /* variantId= */ "testVariant"); + + MddLogData expectedData = + newLogDataBuilderWithClientInfo() + .setSamplingInterval(SAMPLING_ALWAYS) + .setDataDownloadFileGroupStats( + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName("fileGroup") + .setFileGroupVersionNumber(1) + .setBuildId(123) + .setVariantId("testVariant")) + .setDeviceInfo(MddDeviceInfo.newBuilder().setDeviceStorageLow(false)) + .setStableSamplingInfo(getStableSamplingInfo()) + .build(); + + verify(mockLogger).log(expectedData, MddClientEvent.Code.EVENT_CODE_UNSPECIFIED_VALUE); + } + + @Test + public void testLogExpirationHandlerRemoveUnaccountedFilesSampled() { + final int unaccountedFileCount = 5; + overrideDefaultSampleInterval(SAMPLING_ALWAYS); + mddEventLogger.logMddDataDownloadFileExpirationEvent(0, unaccountedFileCount); + + MddLogData expectedData = + newLogDataBuilderWithClientInfo() + .setSamplingInterval(SAMPLING_ALWAYS) + .setDeviceInfo(MddDeviceInfo.newBuilder().setDeviceStorageLow(false)) + .setStableSamplingInfo(getStableSamplingInfo()) + .build(); + + verify(mockLogger).log(expectedData, MddClientEvent.Code.EVENT_CODE_UNSPECIFIED_VALUE); + } + + @Test + public void testLogMddNetworkSavingsSampled() { + overrideDefaultSampleInterval(SAMPLING_ALWAYS); + DataDownloadFileGroupStats icingDataDownloadFileGroupStats = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName("fileGroup") + .setFileGroupVersionNumber(1) + .build(); + mddEventLogger.logMddNetworkSavings( + icingDataDownloadFileGroupStats, 0, 200L, 100L, "file-id", 1); + MddLogData expectedData = + newLogDataBuilderWithClientInfo() + .setSamplingInterval(SAMPLING_ALWAYS) + .setDeviceInfo(MddDeviceInfo.newBuilder().setDeviceStorageLow(false)) + .setStableSamplingInfo(getStableSamplingInfo()) + .build(); + + verify(mockLogger).log(expectedData, MddClientEvent.Code.EVENT_CODE_UNSPECIFIED_VALUE); + } + + @Test + public void testLogMddDownloadResult() { + overrideDefaultSampleInterval(SAMPLING_ALWAYS); + DataDownloadFileGroupStats icingDataDownloadFileGroupStats = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName("fileGroup") + .setFileGroupVersionNumber(1) + .build(); + mddEventLogger.logMddDownloadResult( + MddDownloadResult.Code.LOW_DISK_ERROR, icingDataDownloadFileGroupStats); + + MddLogData expectedData = + newLogDataBuilderWithClientInfo() + .setSamplingInterval(SAMPLING_ALWAYS) + .setMddDownloadResultLog( + MddDownloadResultLog.newBuilder() + .setResult(MddDownloadResult.Code.LOW_DISK_ERROR) + .setDataDownloadFileGroupStats(icingDataDownloadFileGroupStats)) + .setDeviceInfo(MddDeviceInfo.newBuilder().setDeviceStorageLow(false)) + .setStableSamplingInfo(getStableSamplingInfo()) + .build(); + + verify(mockLogger).log(expectedData, MddClientEvent.Code.DATA_DOWNLOAD_RESULT_LOG_VALUE); + } + + @Test + public void testLogMddUsageEvent() { + overrideDefaultSampleInterval(SAMPLING_ALWAYS); + + DataDownloadFileGroupStats icingDataDownloadFileGroupStats = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName("fileGroup") + .setFileGroupVersionNumber(1) + .setBuildId(123) + .setVariantId("variant-id") + .build(); + + Void usageEventLog = null; + + mddEventLogger.logMddUsageEvent(icingDataDownloadFileGroupStats, usageEventLog); + + MddLogData expectedData = + newLogDataBuilderWithClientInfo() + .setDataDownloadFileGroupStats(icingDataDownloadFileGroupStats) + .setSamplingInterval(SAMPLING_ALWAYS) + .setDeviceInfo(MddDeviceInfo.newBuilder().setDeviceStorageLow(false)) + .setStableSamplingInfo(getStableSamplingInfo()) + .build(); + + verify(mockLogger).log(expectedData, MddClientEvent.Code.EVENT_CODE_UNSPECIFIED_VALUE); + } + + @Test + public void testlogMddLibApiResultLog() { + overrideApiLoggingSampleInterval(SAMPLING_ALWAYS); + + DataDownloadFileGroupStats icingDataDownloadFileGroupStats = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName("fileGroup") + .setFileGroupVersionNumber(1) + .build(); + + Void mddLibApiResultLog = null; + mddEventLogger.logMddLibApiResultLog(mddLibApiResultLog); + + MddLogData expectedData = + newLogDataBuilderWithClientInfo() + .setSamplingInterval(SAMPLING_ALWAYS) + .setDeviceInfo(MddDeviceInfo.newBuilder().setDeviceStorageLow(false)) + .setStableSamplingInfo(getStableSamplingInfo()) + .build(); + + verify(mockLogger).log(expectedData, MddClientEvent.Code.EVENT_CODE_UNSPECIFIED_VALUE); + } + + private void overrideDefaultSampleInterval(int sampleInterval) { + flags.mddDefaultSampleInterval = Optional.of(sampleInterval); + } + + private void overrideApiLoggingSampleInterval(int sampleInterval) { + flags.apiLoggingSampleInterval = Optional.of(sampleInterval); + } + + private StableSamplingInfo getStableSamplingInfo() { + if (flags.enableRngBasedDeviceStableSampling()) { + return StableSamplingInfo.newBuilder() + .setStableSamplingUsed(true) + .setStableSamplingFirstEnabledTimestampMs(0) + .setPartOfAlwaysLoggingGroup(false) + .setInvalidSamplingRateUsed(false) + .build(); + } + + return StableSamplingInfo.newBuilder().setStableSamplingUsed(false).build(); + } +} diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/NetworkLoggerTest.java b/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/NetworkLoggerTest.java new file mode 100644 index 0000000..a84f537 --- /dev/null +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/NetworkLoggerTest.java @@ -0,0 +1,170 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.internal.logging; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.util.concurrent.Futures.immediateVoidFuture; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import android.content.Context; +import androidx.test.core.app.ApplicationProvider; +import com.google.mobiledatadownload.internal.MetadataProto.FileGroupLoggingState; +import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; +import com.google.android.libraries.mobiledatadownload.file.common.testing.TemporaryUri; +import com.google.android.libraries.mobiledatadownload.testing.FakeTimeSource; +import com.google.android.libraries.mobiledatadownload.testing.MddTestDependencies; +import com.google.android.libraries.mobiledatadownload.testing.TestFlags; +import com.google.common.base.Optional; +import com.google.common.util.concurrent.AsyncCallable; +import java.util.Random; +import java.util.concurrent.Executor; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.robolectric.RobolectricTestRunner; + +@RunWith(RobolectricTestRunner.class) +public class NetworkLoggerTest { + + private static final String GROUP_NAME_1 = "group-name-1"; + private static final String OWNER_PACKAGE_1 = "owner-package-1"; + private static final int VERSION_NUMBER_1 = 1; + private static final int BUILD_ID_1 = 1; + + private static final String GROUP_NAME_2 = "group-name-2"; + private static final String OWNER_PACKAGE_2 = "owner-package-2"; + private static final int VERSION_NUMBER_2 = 2; + private static final int BUILD_ID_2 = 1; + + private static final String GROUP_NAME_3 = "group-name-3"; + private static final String OWNER_PACKAGE_3 = "owner-package-3"; + private static final int VERSION_NUMBER_3 = 3; + private static final int BUILD_ID_3 = 1; + private static final Executor executor = directExecutor(); + + private final Context context = ApplicationProvider.getApplicationContext(); + + private final TestFlags flags = new TestFlags(); + + private LoggingStateStore loggingStateStore; + @Mock EventLogger mockEventLogger; + + @Rule public final TemporaryUri tmpUri = new TemporaryUri(); + @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + + @Captor ArgumentCaptor<AsyncCallable<Void>> mddNetworkStatsArgumentCaptor; + + @Before + public void setUp() throws Exception { + loggingStateStore = + MddTestDependencies.LoggingStateStoreImpl.SHARED_PREFERENCES.loggingStateStore( + context, Optional.absent(), new FakeTimeSource(), executor, new Random()); + } + + @Test + public void testLogNetworkStats_log() throws Exception { + flags.networkStatsLoggingSampleInterval = Optional.of(1); + + setupNetworkUsage(); + + NetworkLogger networkLogger = + new NetworkLogger(context, mockEventLogger, Optional.absent(), flags, loggingStateStore); + when(mockEventLogger.logMddNetworkStats(any())).thenReturn(immediateVoidFuture()); + networkLogger.log().get(); + + verify(mockEventLogger, times(1)).logMddNetworkStats(mddNetworkStatsArgumentCaptor.capture()); + + // Verify that all entries are cleared after logging. + verifyAllEntriesAreCleared(); + } + + private void verifyAllEntriesAreCleared() throws Exception { + assertThat(loggingStateStore.getAndResetAllDataUsage().get()).isEmpty(); + } + + @Test + public void testLogNetworkStats_noNetworkUsage_logsNoUsage() throws Exception { + flags.networkStatsLoggingSampleInterval = Optional.of(1); + + NetworkLogger networkLogger = + new NetworkLogger(context, mockEventLogger, Optional.absent(), flags, loggingStateStore); + when(mockEventLogger.logMddNetworkStats(any())).thenReturn(immediateVoidFuture()); + + networkLogger.log().get(); + + verify(mockEventLogger, times(1)).logMddNetworkStats(mddNetworkStatsArgumentCaptor.capture()); + + // Verify that all entries are cleared after logging. + verifyAllEntriesAreCleared(); + } + + private void setupNetworkUsage() throws Exception { + loggingStateStore + .incrementDataUsage( + FileGroupLoggingState.newBuilder() + .setGroupKey( + GroupKey.newBuilder() + .setGroupName(GROUP_NAME_1) + .setOwnerPackage(OWNER_PACKAGE_1) + .build()) + .setFileGroupVersionNumber(VERSION_NUMBER_1) + .setBuildId(BUILD_ID_1) + .setWifiUsage(1) + .setCellularUsage(2) + .build()) + .get(); + + loggingStateStore + .incrementDataUsage( + FileGroupLoggingState.newBuilder() + .setGroupKey( + GroupKey.newBuilder() + .setGroupName(GROUP_NAME_2) + .setOwnerPackage(OWNER_PACKAGE_2) + .build()) + .setFileGroupVersionNumber(VERSION_NUMBER_2) + .setBuildId(BUILD_ID_2) + .setWifiUsage(4) + .setCellularUsage(0) + .build()) + .get(); + + loggingStateStore + .incrementDataUsage( + FileGroupLoggingState.newBuilder() + .setGroupKey( + GroupKey.newBuilder() + .setGroupName(GROUP_NAME_3) + .setOwnerPackage(OWNER_PACKAGE_3) + .build()) + .setFileGroupVersionNumber(VERSION_NUMBER_3) + .setBuildId(BUILD_ID_3) + .setWifiUsage(0) + .setCellularUsage(8) + .build()) + .get(); + } +} diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/StorageLoggerTest.java b/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/StorageLoggerTest.java new file mode 100644 index 0000000..e53be67 --- /dev/null +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/StorageLoggerTest.java @@ -0,0 +1,838 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.internal.logging; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.util.concurrent.Futures.immediateVoidFuture; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import android.content.Context; +import android.net.Uri; +import androidx.test.core.app.ApplicationProvider; +import com.google.mobiledatadownload.internal.MetadataProto.DataFile; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; +import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; +import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey; +import com.google.android.libraries.mobiledatadownload.SilentFeedback; +import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; +import com.google.android.libraries.mobiledatadownload.file.spi.Backend; +import com.google.android.libraries.mobiledatadownload.internal.FileGroupsMetadata; +import com.google.android.libraries.mobiledatadownload.internal.MddConstants; +import com.google.android.libraries.mobiledatadownload.internal.MddTestUtil; +import com.google.android.libraries.mobiledatadownload.internal.SharedFileManager; +import com.google.android.libraries.mobiledatadownload.internal.SharedFilesMetadata; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup; +import com.google.android.libraries.mobiledatadownload.internal.logging.StorageLogger.GroupStorage; +import com.google.android.libraries.mobiledatadownload.internal.util.DirectoryUtil; +import com.google.android.libraries.mobiledatadownload.testing.TestFlags; +import com.google.auto.value.AutoValue; +import com.google.common.base.Optional; +import com.google.common.util.concurrent.AsyncCallable; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; +import com.google.mobiledatadownload.LogProto.MddStorageStats; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.robolectric.RobolectricTestRunner; + +@RunWith(RobolectricTestRunner.class) +public class StorageLoggerTest { + private static final String GROUP_1 = "group1"; + private static final String GROUP_2 = "group2"; + private static final String PACKAGE_1 = "package1"; + private static final String PACKAGE_2 = "package2"; + private static final int FILE_GROUP_VERSION_NUMBER_1 = 10; + private static final int FILE_GROUP_VERSION_NUMBER_2 = 20; + + private static final long BUILD_ID_1 = 10; + private static final long BUILD_ID_2 = 20; + private static final String VARIANT_ID = "test-variant"; + + // Note: We can't make those android uris static variable since the Uri.parse will fail + // with initialization. + private final Uri androidUri1 = + Uri.parse("android://com.google.android.gms/files/datadownload/shared/public/file_1"); + private static final long FILE_SIZE_1 = 1; + + private final Uri androidUri2 = + Uri.parse("android://com.google.android.gms/files/datadownload/shared/public/file_2"); + private static final long FILE_SIZE_2 = 2; + + private final Uri androidUri3 = + Uri.parse("android://com.google.android.gms/files/datadownload/shared/public/file_3"); + private static final long FILE_SIZE_3 = 4; + + private final Uri androidUri4 = + Uri.parse("android://com.google.android.gms/files/datadownload/shared/public/file_4"); + private static final long FILE_SIZE_4 = 8; + + private final Uri androidUri5 = + Uri.parse("android://com.google.android.gms/files/datadownload/shared/public/file_5"); + private static final long FILE_SIZE_5 = 16; + + private final Uri androidUri6 = + Uri.parse("android://com.google.android.gms/files/datadownload/shared/public/file_6"); + private static final long FILE_SIZE_6 = 32; + + private final Uri inlineUri1 = + Uri.parse("android://com.google.android.gms/files/datadownload/shared/public/inline_file_1"); + private static final long INLINE_FILE_SIZE_1 = 64; + + private static final long MDD_DIRECTORY_SIZE = + FILE_SIZE_1 + + FILE_SIZE_2 + + FILE_SIZE_3 + + FILE_SIZE_4 + + FILE_SIZE_5 + + FILE_SIZE_6 + + INLINE_FILE_SIZE_1; + + // These files will belong to 2 groups + private static final DataFile DATA_FILE_1 = MddTestUtil.createDataFile("file1", 1); + private static final DataFile DATA_FILE_2 = MddTestUtil.createDataFile("file2", 2); + private static final DataFile DATA_FILE_3 = MddTestUtil.createDataFile("file3", 3); + private static final DataFile DATA_FILE_4 = MddTestUtil.createDataFile("file4", 4); + private static final DataFile DATA_FILE_5 = MddTestUtil.createDataFile("file5", 5); + private static final DataFile DATA_FILE_6 = MddTestUtil.createDataFile("file6", 6); + private static final DataFile INLINE_DATA_FILE_1 = + DataFile.newBuilder() + .setFileId("inlineFile1") + .setUrlToDownload("inlinefile:sha1:inlinefile1") + .setChecksum("inlinefile1") + .setByteSize((int) INLINE_FILE_SIZE_1) + .build(); + + private SynchronousFileStorage fileStorage; + + private final Context context = ApplicationProvider.getApplicationContext(); + + @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + + @Mock EventLogger mockEventLogger; + @Mock FileGroupsMetadata mockFileGroupsMetadata; + @Mock SharedFileManager mockSharedFileManager; + @Mock Backend mockBackend; + @Mock SilentFeedback mockSilentFeedback; + + @Captor ArgumentCaptor<AsyncCallable<MddStorageStats>> mddStorageStatsCallableArgumentCaptor; + + private final TestFlags flags = new TestFlags(); + + @Before + public void setUp() throws Exception { + + setUpFileMock(androidUri1, FILE_SIZE_1); + setUpFileMock(androidUri2, FILE_SIZE_2); + setUpFileMock(androidUri3, FILE_SIZE_3); + setUpFileMock(androidUri4, FILE_SIZE_4); + setUpFileMock(androidUri5, FILE_SIZE_5); + setUpFileMock(androidUri6, FILE_SIZE_6); + setUpFileMock(inlineUri1, INLINE_FILE_SIZE_1); + + Uri downloadDirUri = DirectoryUtil.getBaseDownloadDirectory(context, Optional.absent()); + setUpDirectoryMock( + downloadDirUri, + Arrays.asList( + androidUri1, + androidUri2, + androidUri3, + androidUri4, + androidUri5, + androidUri6, + inlineUri1)); + + when(mockBackend.name()).thenReturn("android"); + fileStorage = new SynchronousFileStorage(Arrays.asList(mockBackend)); + + flags.storageStatsLoggingSampleInterval = Optional.of(1); + } + + // TODO(b/115659980): consider moving this to a public utility class in the File Library + private void setUpFileMock(Uri uri, long size) throws IOException { + when(mockBackend.exists(uri)).thenReturn(true); + when(mockBackend.isDirectory(uri)).thenReturn(false); + when(mockBackend.fileSize(uri)).thenReturn(size); + } + + // TODO(b/115659980): consider moving this to a public utility class in the File Library + private void setUpDirectoryMock(Uri uri, List<Uri> children) throws IOException { + when(mockBackend.exists(uri)).thenReturn(true); + when(mockBackend.isDirectory(uri)).thenReturn(true); + when(mockBackend.children(uri)).thenReturn(children); + } + + @Test + public void testLogMddStorageStats() throws Exception { + // Setup Group1 that has 3 FileDataGroups: + // - Stale group has DATA_FILE_1, DATA_FILE_2. + // - Downloaded group has DATA_FILE_2, DATA_FILE_3. + // - Pending group has DATA_FILE_3, DATA_FILE_4. + DataFileGroupInternal group1Stale = + createDataFileGroupWithFiles( + GROUP_1, + PACKAGE_1, + Arrays.asList(DATA_FILE_1, DATA_FILE_2), + Arrays.asList(androidUri1, androidUri2)); + DataFileGroupInternal group1Downloaded = + createDataFileGroupWithFiles( + GROUP_1, + PACKAGE_1, + Arrays.asList(DATA_FILE_2, DATA_FILE_3), + Arrays.asList(androidUri2, androidUri3)) + .toBuilder() + .setFileGroupVersionNumber(FILE_GROUP_VERSION_NUMBER_1) + .setBuildId(BUILD_ID_1) + .setVariantId(VARIANT_ID) + .build(); + DataFileGroupInternal group1Pending = + createDataFileGroupWithFiles( + GROUP_1, + PACKAGE_1, + Arrays.asList(DATA_FILE_3, DATA_FILE_4), + Arrays.asList(androidUri3, androidUri4)); + + // Setup Group2 that has 2 FileDataGroups: + // - Downloaded group has DATA_FILE_5. + // - Pending group has DATA_FILE_6. + DataFileGroupInternal group2Downloaded = + createDataFileGroupWithFiles( + GROUP_2, PACKAGE_2, Arrays.asList(DATA_FILE_5), Arrays.asList(androidUri5)) + .toBuilder() + .setFileGroupVersionNumber(FILE_GROUP_VERSION_NUMBER_2) + .setBuildId(BUILD_ID_2) + .setVariantId(VARIANT_ID) + .build(); + DataFileGroupInternal group2Pending = + createDataFileGroupWithFiles( + GROUP_2, PACKAGE_2, Arrays.asList(DATA_FILE_6), Arrays.asList(androidUri6)); + + List<GroupKeyAndGroup> groups = new ArrayList<>(); + groups.add(createGroupKeyAndGroup(group1Downloaded, true /*downloaded*/)); + groups.add(createGroupKeyAndGroup(group1Pending, false /*downloaded*/)); + groups.add(createGroupKeyAndGroup(group2Downloaded, true /*downloaded*/)); + groups.add(createGroupKeyAndGroup(group2Pending, false /*downloaded*/)); + when(mockFileGroupsMetadata.getAllFreshGroups()).thenReturn(Futures.immediateFuture(groups)); + + when(mockFileGroupsMetadata.getAllStaleGroups()) + .thenReturn(Futures.immediateFuture(Arrays.asList(group1Stale))); + + verifyStorageStats( + /* totalMddBytesUsed= */ FILE_SIZE_1 + + FILE_SIZE_2 + + FILE_SIZE_3 + + FILE_SIZE_4 + + FILE_SIZE_5 + + FILE_SIZE_6, + ExpectedFileGroupStorageStats.create( + GROUP_1, + PACKAGE_1, + BUILD_ID_1, + VARIANT_ID, + FILE_GROUP_VERSION_NUMBER_1, + createGroupStorage( + /* totalBytesUsed= */ FILE_SIZE_1 + FILE_SIZE_2 + FILE_SIZE_3 + FILE_SIZE_4, + /* totalInlineBytesUsed= */ 0, + /* downloadedGroupBytesUsed= */ FILE_SIZE_2 + FILE_SIZE_3, + /* downloadedGroupInlineBytesUsed= */ 0, + /* totalFileCount= */ 2, + /* totalInlineFileCount= */ 0)), + ExpectedFileGroupStorageStats.create( + GROUP_2, + PACKAGE_2, + BUILD_ID_2, + VARIANT_ID, + FILE_GROUP_VERSION_NUMBER_2, + createGroupStorage( + /* totalBytesUsed= */ FILE_SIZE_5 + FILE_SIZE_6, + /* totalInlineBytesUsed= */ 0, + /* downloadedGroupBytesUsed= */ FILE_SIZE_5, + /* downloadedGroupInlineBytesUsed= */ 0, + /* totalFileCount= */ 1, + /* totalInlineFileCount= */ 0))); + } + + @Test + public void testLogMddStorageStats_noDownloadedInGroup2() throws Exception { + // Setup Group1 that has 3 FileDataGroups: + // - Stale group has DATA_FILE_1, DATA_FILE_2. + // - Downloaded group has DATA_FILE_2, DATA_FILE_3. + // - Pending group has DATA_FILE_3, DATA_FILE_4. + DataFileGroupInternal group1Stale = + createDataFileGroupWithFiles( + GROUP_1, + PACKAGE_1, + Arrays.asList(DATA_FILE_1, DATA_FILE_2), + Arrays.asList(androidUri1, androidUri2)); + DataFileGroupInternal group1Downloaded = + createDataFileGroupWithFiles( + GROUP_1, + PACKAGE_1, + Arrays.asList(DATA_FILE_2, DATA_FILE_3), + Arrays.asList(androidUri2, androidUri3)) + .toBuilder() + .setFileGroupVersionNumber(FILE_GROUP_VERSION_NUMBER_1) + .setBuildId(BUILD_ID_1) + .setVariantId(VARIANT_ID) + .build(); + DataFileGroupInternal group1Pending = + createDataFileGroupWithFiles( + GROUP_1, + PACKAGE_1, + Arrays.asList(DATA_FILE_3, DATA_FILE_4), + Arrays.asList(androidUri3, androidUri4)); + + // Setup Group2 that has 2 FileDataGroups (no downloaded) + // - Stale group has DATA_FILE_5. + // - Pending group has DATA_FILE_6. + DataFileGroupInternal group2Stale = + createDataFileGroupWithFiles( + GROUP_2, PACKAGE_2, Arrays.asList(DATA_FILE_5), Arrays.asList(androidUri5)); + DataFileGroupInternal group2Pending = + createDataFileGroupWithFiles( + GROUP_2, PACKAGE_2, Arrays.asList(DATA_FILE_6), Arrays.asList(androidUri6)); + + List<GroupKeyAndGroup> groups = new ArrayList<>(); + groups.add(createGroupKeyAndGroup(group1Downloaded, true /*downloaded*/)); + groups.add(createGroupKeyAndGroup(group1Pending, false /*downloaded*/)); + groups.add(createGroupKeyAndGroup(group2Pending, false /*downloaded*/)); + when(mockFileGroupsMetadata.getAllFreshGroups()).thenReturn(Futures.immediateFuture(groups)); + + when(mockFileGroupsMetadata.getAllStaleGroups()) + .thenReturn(Futures.immediateFuture(Arrays.asList(group1Stale, group2Stale))); + + verifyStorageStats( + /* totalMddBytesUsed= */ FILE_SIZE_1 + + FILE_SIZE_2 + + FILE_SIZE_3 + + FILE_SIZE_4 + + FILE_SIZE_5 + + FILE_SIZE_6, + ExpectedFileGroupStorageStats.create( + GROUP_1, + PACKAGE_1, + BUILD_ID_1, + VARIANT_ID, + FILE_GROUP_VERSION_NUMBER_1, + createGroupStorage( + /* totalBytesUsed= */ FILE_SIZE_1 + FILE_SIZE_2 + FILE_SIZE_3 + FILE_SIZE_4, + /* totalInlineBytesUsed= */ 0, + /* downloadedGroupBytesUsed= */ FILE_SIZE_2 + FILE_SIZE_3, + /* downloadedGroupInlineBytesUsed= */ 0, + /* totalFileCount= */ 2, + /* totalInlineFileCount= */ 0)), + ExpectedFileGroupStorageStats.create( + GROUP_2, + PACKAGE_2, + /* buildId= */ 0, + /* variantId= */ "", + /* fileGroupVersionNumber= */ -1, + createGroupStorage( + /* totalBytesUsed= */ FILE_SIZE_5 + FILE_SIZE_6, + /* totalInlineBytesUsed= */ 0, + /* downloadedGroupBytesUsed= */ 0, + /* downloadedGroupInlineBytesUsed= */ 0, + /* totalFileCount= */ 1, + /* totalInlineFileCount= */ 0))); + } + + @Test + public void testLogMddStorageStats_commonFilesBetweenGroups() throws Exception { + // Setup Group1 that has 3 FileDataGroups: + // - Stale group has DATA_FILE_1, DATA_FILE_2. + // - Downloaded group has DATA_FILE_2, DATA_FILE_3. + // - Pending group has DATA_FILE_3, DATA_FILE_4. + DataFileGroupInternal group1Stale = + createDataFileGroupWithFiles( + GROUP_1, + PACKAGE_1, + Arrays.asList(DATA_FILE_1, DATA_FILE_2), + Arrays.asList(androidUri1, androidUri2)); + DataFileGroupInternal group1Downloaded = + createDataFileGroupWithFiles( + GROUP_1, + PACKAGE_1, + Arrays.asList(DATA_FILE_2, DATA_FILE_3), + Arrays.asList(androidUri2, androidUri3)) + .toBuilder() + .setFileGroupVersionNumber(FILE_GROUP_VERSION_NUMBER_1) + .setBuildId(BUILD_ID_1) + .setVariantId(VARIANT_ID) + .build(); + DataFileGroupInternal group1Pending = + createDataFileGroupWithFiles( + GROUP_1, + PACKAGE_1, + Arrays.asList(DATA_FILE_3, DATA_FILE_4), + Arrays.asList(androidUri3, androidUri4)); + + // Setup Group2 that has 3 FileDataGroups: + // - Stale group has DATA_FILE_1, DATA_FILE_3. + // - Downloaded group has DATA_FILE_4, DATA_FILE_5. + // - Pending group has DATA_FILE_6. + DataFileGroupInternal group2Stale = + createDataFileGroupWithFiles( + GROUP_2, + PACKAGE_2, + Arrays.asList(DATA_FILE_1, DATA_FILE_3), + Arrays.asList(androidUri1, androidUri3)); + DataFileGroupInternal group2Downloaded = + createDataFileGroupWithFiles( + GROUP_2, + PACKAGE_2, + Arrays.asList(DATA_FILE_4, DATA_FILE_5), + Arrays.asList(androidUri4, androidUri5)) + .toBuilder() + .setFileGroupVersionNumber(FILE_GROUP_VERSION_NUMBER_2) + .setBuildId(BUILD_ID_2) + .setVariantId(VARIANT_ID) + .build(); + DataFileGroupInternal group2Pending = + createDataFileGroupWithFiles( + GROUP_2, PACKAGE_2, Arrays.asList(DATA_FILE_6), Arrays.asList(androidUri6)); + + List<GroupKeyAndGroup> groups = new ArrayList<>(); + groups.add(createGroupKeyAndGroup(group1Downloaded, true /*downloaded*/)); + groups.add(createGroupKeyAndGroup(group1Pending, false /*downloaded*/)); + groups.add(createGroupKeyAndGroup(group2Downloaded, true /*downloaded*/)); + groups.add(createGroupKeyAndGroup(group2Pending, false /*downloaded*/)); + when(mockFileGroupsMetadata.getAllFreshGroups()).thenReturn(Futures.immediateFuture(groups)); + + when(mockFileGroupsMetadata.getAllStaleGroups()) + .thenReturn(Futures.immediateFuture(Arrays.asList(group1Stale, group2Stale))); + + verifyStorageStats( + /* totalMddBytesUsed= */ FILE_SIZE_1 + + FILE_SIZE_2 + + FILE_SIZE_3 + + FILE_SIZE_4 + + FILE_SIZE_5 + + FILE_SIZE_6, + ExpectedFileGroupStorageStats.create( + GROUP_1, + PACKAGE_1, + BUILD_ID_1, + VARIANT_ID, + FILE_GROUP_VERSION_NUMBER_1, + createGroupStorage( + /* totalBytesUsed= */ FILE_SIZE_1 + FILE_SIZE_2 + FILE_SIZE_3 + FILE_SIZE_4, + /* totalInlineBytesUsed= */ 0, + /* downloadedGroupBytesUsed= */ FILE_SIZE_2 + FILE_SIZE_3, + /* downloadedGroupInlineBytesUsed= */ 0, + /* totalFileCount= */ 2, + /* totalInlineFileCount= */ 0)), + ExpectedFileGroupStorageStats.create( + GROUP_2, + PACKAGE_2, + BUILD_ID_2, + VARIANT_ID, + FILE_GROUP_VERSION_NUMBER_2, + createGroupStorage( + /* totalBytesUsed= */ FILE_SIZE_1 + + FILE_SIZE_3 + + FILE_SIZE_4 + + FILE_SIZE_5 + + FILE_SIZE_6, + /* totalInlineBytesUsed= */ 0, + /* downloadedGroupBytesUsed= */ FILE_SIZE_4 + FILE_SIZE_5, + /* downloadedGroupInlineBytesUsed= */ 0, + /* totalFileCount= */ 2, + /* totalInlineFileCount= */ 0))); + } + + @Test + public void testLogMddStorageStats_emptyDownloadedGroup() throws Exception { + // Setup Group1 that has 3 FileDataGroups: + // - Stale group has DATA_FILE_1, DATA_FILE_2. + // - Downloaded group has DATA_FILE_2, DATA_FILE_3. + // - Pending group has DATA_FILE_3, DATA_FILE_4. + DataFileGroupInternal group1Stale = + createDataFileGroupWithFiles( + GROUP_1, + PACKAGE_1, + Arrays.asList(DATA_FILE_1, DATA_FILE_2), + Arrays.asList(androidUri1, androidUri2)); + DataFileGroupInternal group1Downloaded = + createDataFileGroupWithFiles( + GROUP_1, + PACKAGE_1, + Arrays.asList(DATA_FILE_2, DATA_FILE_3), + Arrays.asList(androidUri2, androidUri3)) + .toBuilder() + .setFileGroupVersionNumber(FILE_GROUP_VERSION_NUMBER_1) + .setBuildId(BUILD_ID_1) + .setVariantId(VARIANT_ID) + .build(); + DataFileGroupInternal group1Pending = + createDataFileGroupWithFiles( + GROUP_1, + PACKAGE_1, + Arrays.asList(DATA_FILE_3, DATA_FILE_4), + Arrays.asList(androidUri3, androidUri4)); + + // Downloaded Group2 is empty (no file). This could happen when we send an empty FileGroup to + // clear old config. + DataFileGroupInternal group2Downloaded = + createDataFileGroupWithFiles( + GROUP_2, PACKAGE_2, new ArrayList<>() /*dataFiles*/, new ArrayList<>() /*fileUris*/) + .toBuilder() + .setFileGroupVersionNumber(FILE_GROUP_VERSION_NUMBER_2) + .setBuildId(BUILD_ID_2) + .setVariantId(VARIANT_ID) + .build(); + + List<GroupKeyAndGroup> groups = new ArrayList<>(); + groups.add(createGroupKeyAndGroup(group1Downloaded, true /*downloaded*/)); + groups.add(createGroupKeyAndGroup(group1Pending, false /*downloaded*/)); + groups.add(createGroupKeyAndGroup(group2Downloaded, true /*downloaded*/)); + when(mockFileGroupsMetadata.getAllFreshGroups()).thenReturn(Futures.immediateFuture(groups)); + + when(mockFileGroupsMetadata.getAllStaleGroups()) + .thenReturn(Futures.immediateFuture(Arrays.asList(group1Stale))); + + verifyStorageStats( + /* totalMddBytesUsed= */ FILE_SIZE_1 + FILE_SIZE_2 + FILE_SIZE_3 + FILE_SIZE_4, + ExpectedFileGroupStorageStats.create( + GROUP_1, + PACKAGE_1, + BUILD_ID_1, + VARIANT_ID, + FILE_GROUP_VERSION_NUMBER_1, + createGroupStorage( + /* totalBytesUsed= */ FILE_SIZE_1 + FILE_SIZE_2 + FILE_SIZE_3 + FILE_SIZE_4, + /* totalInlineBytesUsed= */ 0, + /* downloadedGroupBytesUsed= */ FILE_SIZE_2 + FILE_SIZE_3, + /* downloadedGroupInlineBytesUsed= */ 0, + /* totalFileCount= */ 2, + /* totalInlineFileCount= */ 0)), + ExpectedFileGroupStorageStats.create( + GROUP_2, + PACKAGE_2, + BUILD_ID_2, + VARIANT_ID, + FILE_GROUP_VERSION_NUMBER_2, + createGroupStorage( + /* totalBytesUsed= */ 0, + /* totalInlineBytesUsed= */ 0, + /* downloadedGroupBytesUsed= */ 0, + /* downloadedGroupInlineBytesUsed= */ 0, + /* totalFileCount= */ 0, + /* totalInlineFileCount= */ 0))); + } + + @Test + public void testLogMddStorageStats_mddDirectoryNotExists() throws Exception { + when(mockFileGroupsMetadata.getAllFreshGroups()) + .thenReturn(Futures.immediateFuture(new ArrayList<>())); + when(mockFileGroupsMetadata.getAllStaleGroups()) + .thenReturn(Futures.immediateFuture(new ArrayList<>())); + when(mockBackend.exists(DirectoryUtil.getBaseDownloadDirectory(context, Optional.absent()))) + .thenReturn(false); + + StorageLogger storageLogger = + new StorageLogger( + context, + mockFileGroupsMetadata, + mockSharedFileManager, + fileStorage, + mockEventLogger, + mockSilentFeedback, + Optional.absent(), + MoreExecutors.directExecutor()); + + when(mockEventLogger.logMddStorageStats(any())).thenReturn(immediateVoidFuture()); + + storageLogger.logStorageStats(/* daysSinceLastLog= */ 1).get(); + + verify(mockEventLogger, times(1)) + .logMddStorageStats(mddStorageStatsCallableArgumentCaptor.capture()); + AsyncCallable<MddStorageStats> mddStorageStatsCallable = + mddStorageStatsCallableArgumentCaptor.getValue(); + + MddStorageStats mddStorageStats = mddStorageStatsCallable.call().get(); + assertThat(mddStorageStats.getTotalMddBytesUsed()).isEqualTo(0); + assertThat(mddStorageStats.getTotalMddDirectoryBytesUsed()).isEqualTo(0); + + assertThat(mddStorageStats.getDataDownloadFileGroupStatsList()).isEmpty(); + assertThat(mddStorageStats.getTotalBytesUsedList()).isEmpty(); + assertThat(mddStorageStats.getDownloadedGroupBytesUsedList()).isEmpty(); + } + + @Test + public void testMddStorageStats_includesDaysSinceLastLog() throws Exception { + when(mockFileGroupsMetadata.getAllFreshGroups()) + .thenReturn(Futures.immediateFuture(new ArrayList<>())); + when(mockFileGroupsMetadata.getAllStaleGroups()) + .thenReturn(Futures.immediateFuture(new ArrayList<>())); + when(mockBackend.exists(DirectoryUtil.getBaseDownloadDirectory(context, Optional.absent()))) + .thenReturn(false); + + StorageLogger storageLogger = + new StorageLogger( + context, + mockFileGroupsMetadata, + mockSharedFileManager, + fileStorage, + mockEventLogger, + mockSilentFeedback, + Optional.absent(), + MoreExecutors.directExecutor()); + + when(mockEventLogger.logMddStorageStats(any())).thenReturn(immediateVoidFuture()); + + storageLogger.logStorageStats(/* daysSinceLastLog= */ -1).get(); + + verify(mockEventLogger, times(1)) + .logMddStorageStats(mddStorageStatsCallableArgumentCaptor.capture()); + + AsyncCallable<MddStorageStats> mddStorageStatsCallable = + mddStorageStatsCallableArgumentCaptor.getValue(); + MddStorageStats mddStorageStats = mddStorageStatsCallable.call().get(); + + assertThat(mddStorageStats.getDaysSinceLastLog()).isEqualTo(-1); + } + + @Test + public void testLogMddStorageStats_groupWithInlineFiles() throws Exception { + // Setup Group1 that has 3 FileDataGroups: + // - Stale group has DATA_FILE_1, DATA_FILE_2. + // - Downloaded group has DATA_FILE_2, INLINE_FILE_1, + // - Pending group has DATA_FILE_3, INLINE_FILE_1, + DataFileGroupInternal group1Stale = + createDataFileGroupWithFiles( + GROUP_1, + PACKAGE_1, + Arrays.asList(DATA_FILE_1, DATA_FILE_2), + Arrays.asList(androidUri1, androidUri2)) + .toBuilder() + .setFileGroupVersionNumber(FILE_GROUP_VERSION_NUMBER_1) + .setBuildId(BUILD_ID_1) + .setVariantId(VARIANT_ID) + .build(); + DataFileGroupInternal group1Downloaded = + createDataFileGroupWithFiles( + GROUP_1, + PACKAGE_1, + Arrays.asList(DATA_FILE_2, INLINE_DATA_FILE_1), + Arrays.asList(androidUri2, inlineUri1)) + .toBuilder() + .setFileGroupVersionNumber(FILE_GROUP_VERSION_NUMBER_1) + .setBuildId(BUILD_ID_1) + .setVariantId(VARIANT_ID) + .build(); + DataFileGroupInternal group1Pending = + createDataFileGroupWithFiles( + GROUP_1, + PACKAGE_1, + Arrays.asList(DATA_FILE_3, INLINE_DATA_FILE_1), + Arrays.asList(androidUri3, inlineUri1)) + .toBuilder() + .setFileGroupVersionNumber(FILE_GROUP_VERSION_NUMBER_1) + .setBuildId(BUILD_ID_1) + .setVariantId(VARIANT_ID) + .build(); + + List<GroupKeyAndGroup> groups = new ArrayList<>(); + groups.add(createGroupKeyAndGroup(group1Downloaded, true /*downloaded*/)); + groups.add(createGroupKeyAndGroup(group1Pending, false /*downloaded*/)); + when(mockFileGroupsMetadata.getAllFreshGroups()).thenReturn(Futures.immediateFuture(groups)); + + when(mockFileGroupsMetadata.getAllStaleGroups()) + .thenReturn(Futures.immediateFuture(Arrays.asList(group1Stale))); + + verifyStorageStats( + /* totalMddBytesUsed= */ FILE_SIZE_1 + FILE_SIZE_2 + FILE_SIZE_3 + INLINE_FILE_SIZE_1, + ExpectedFileGroupStorageStats.create( + GROUP_1, + PACKAGE_1, + BUILD_ID_1, + VARIANT_ID, + FILE_GROUP_VERSION_NUMBER_1, + createGroupStorage( + /* totalBytesUsed= */ FILE_SIZE_1 + FILE_SIZE_2 + FILE_SIZE_3 + INLINE_FILE_SIZE_1, + /* totalInlineBytesUsed= */ INLINE_FILE_SIZE_1, + /* downloadedGroupBytesUsed= */ FILE_SIZE_2 + INLINE_FILE_SIZE_1, + /* downloadedGroupInlineBytesUsed= */ INLINE_FILE_SIZE_1, + /* totalFileCount= */ 2, + /* totalInlineFileCount= */ 1))); + } + + private void verifyStorageStats( + long totalMddBytesUsed, ExpectedFileGroupStorageStats... expectedStatsList) throws Exception { + StorageLogger storageLogger = + new StorageLogger( + context, + mockFileGroupsMetadata, + mockSharedFileManager, + fileStorage, + mockEventLogger, + mockSilentFeedback, + Optional.absent(), + MoreExecutors.directExecutor()); + when(mockEventLogger.logMddStorageStats(any())).thenReturn(immediateVoidFuture()); + storageLogger.logStorageStats(/* daysSinceLastLog= */ 1).get(); + + verify(mockEventLogger, times(1)) + .logMddStorageStats(mddStorageStatsCallableArgumentCaptor.capture()); + + AsyncCallable<MddStorageStats> mddStorageStatsCallable = + mddStorageStatsCallableArgumentCaptor.getValue(); + MddStorageStats mddStorageStats = mddStorageStatsCallable.call().get(); + + assertThat(mddStorageStats.getTotalMddBytesUsed()).isEqualTo(totalMddBytesUsed); + assertThat(mddStorageStats.getTotalMddDirectoryBytesUsed()).isEqualTo(MDD_DIRECTORY_SIZE); + + assertThat(mddStorageStats.getDataDownloadFileGroupStatsCount()) + .isEqualTo(expectedStatsList.length); + assertThat(mddStorageStats.getTotalBytesUsedCount()).isEqualTo(expectedStatsList.length); + assertThat(mddStorageStats.getDownloadedGroupBytesUsedCount()) + .isEqualTo(expectedStatsList.length); + + for (int i = 0; i < expectedStatsList.length; i++) { + DataDownloadFileGroupStats fileGroupStats = + mddStorageStats.getDataDownloadFileGroupStatsList().get(i); + long totalBytesUsed = mddStorageStats.getTotalBytesUsed(i); + long totalInlineBytesUsed = mddStorageStats.getTotalInlineBytesUsed(i); + long downloadedGroupBytesUsed = mddStorageStats.getDownloadedGroupBytesUsed(i); + long downloadedGroupInlineBytesUsed = mddStorageStats.getDownloadedGroupInlineBytesUsed(i); + + ExpectedFileGroupStorageStats expectedStats = + getExpectedStatsForName(fileGroupStats.getFileGroupName(), expectedStatsList); + GroupStorage expectedGroupStorage = expectedStats.groupStorage(); + + assertThat(fileGroupStats.getOwnerPackage()).isEqualTo(expectedStats.packageName()); + assertThat(fileGroupStats.getFileGroupVersionNumber()) + .isEqualTo(expectedStats.fileGroupVersionNumber()); + assertThat(fileGroupStats.getVariantId()).isEqualTo(expectedStats.variantId()); + assertThat(fileGroupStats.getBuildId()).isEqualTo(expectedStats.buildId()); + assertThat(totalBytesUsed).isEqualTo(expectedGroupStorage.totalBytesUsed); + assertThat(totalInlineBytesUsed).isEqualTo(expectedGroupStorage.totalInlineBytesUsed); + assertThat(downloadedGroupBytesUsed).isEqualTo(expectedGroupStorage.downloadedGroupBytesUsed); + assertThat(downloadedGroupInlineBytesUsed) + .isEqualTo(expectedGroupStorage.downloadedGroupInlineBytesUsed); + assertThat(fileGroupStats.getFileCount()).isEqualTo(expectedGroupStorage.totalFileCount); + assertThat(fileGroupStats.getInlineFileCount()) + .isEqualTo(expectedGroupStorage.totalInlineFileCount); + } + } + + /** Find the expected stats for a given group name. */ + private ExpectedFileGroupStorageStats getExpectedStatsForName( + String groupName, ExpectedFileGroupStorageStats[] expectedStatsList) { + for (int i = 0; i < expectedStatsList.length; i++) { + if (groupName.equals(expectedStatsList[i].groupName())) { + return expectedStatsList[i]; + } + } + + throw new AssertionError(String.format("Couldn't find group for name: %s", groupName)); + } + + /** Creates a data file group with the given list of files. */ + private DataFileGroupInternal createDataFileGroupWithFiles( + String fileGroupName, String ownerPackage, List<DataFile> dataFiles, List<Uri> fileUris) { + DataFileGroupInternal.Builder dataFileGroup = + DataFileGroupInternal.newBuilder() + .setGroupName(fileGroupName) + .setOwnerPackage(ownerPackage); + + for (int i = 0; i < dataFiles.size(); ++i) { + DataFile file = dataFiles.get(i); + NewFileKey newFileKey = + SharedFilesMetadata.createKeyFromDataFile(file, dataFileGroup.getAllowedReadersEnum()); + dataFileGroup.addFile(file); + when(mockSharedFileManager.getOnDeviceUri(newFileKey)) + .thenReturn(Futures.immediateFuture(fileUris.get(i))); + } + return dataFileGroup.build(); + } + + private static GroupKeyAndGroup createGroupKeyAndGroup( + DataFileGroupInternal fileGroup, boolean downloaded) { + GroupKey groupKey = createGroupKey(fileGroup, downloaded); + return GroupKeyAndGroup.create(groupKey, fileGroup); + } + + private static GroupKey createGroupKey(DataFileGroupInternal fileGroup, boolean downloaded) { + GroupKey.Builder groupKey = GroupKey.newBuilder().setGroupName(fileGroup.getGroupName()); + + if (fileGroup.getOwnerPackage().isEmpty()) { + groupKey.setOwnerPackage(MddConstants.GMS_PACKAGE); + } else { + groupKey.setOwnerPackage(fileGroup.getOwnerPackage()); + } + groupKey.setDownloaded(downloaded); + + return groupKey.build(); + } + + private static GroupStorage createGroupStorage( + long totalBytesUsed, + long totalInlineBytesUsed, + long downloadedGroupBytesUsed, + long downloadedGroupInlineBytesUsed, + int totalFileCount, + int totalInlineFileCount) { + GroupStorage groupStorage = new GroupStorage(); + groupStorage.totalBytesUsed = totalBytesUsed; + groupStorage.totalInlineBytesUsed = totalInlineBytesUsed; + groupStorage.downloadedGroupBytesUsed = downloadedGroupBytesUsed; + groupStorage.downloadedGroupInlineBytesUsed = downloadedGroupInlineBytesUsed; + groupStorage.totalFileCount = totalFileCount; + groupStorage.totalInlineFileCount = totalInlineFileCount; + return groupStorage; + } + + @AutoValue + abstract static class ExpectedFileGroupStorageStats { + abstract String groupName(); + + abstract String packageName(); + + abstract long buildId(); + + abstract String variantId(); + + abstract int fileGroupVersionNumber(); + + abstract GroupStorage groupStorage(); + + static ExpectedFileGroupStorageStats create( + String groupName, + String packageName, + long buildId, + String variantId, + int fileGroupVersionNumber, + GroupStorage groupStorage) { + return new AutoValue_StorageLoggerTest_ExpectedFileGroupStorageStats( + groupName, packageName, buildId, variantId, fileGroupVersionNumber, groupStorage); + } + } +} diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/util/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/internal/util/BUILD index 36f4805..09c5a02 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/internal/util/BUILD +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/util/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library", "android_local_test") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -30,6 +31,7 @@ android_local_test( "//java/com/google/android/libraries/mobiledatadownload/internal/util:DirectoryUtil", "@androidx_test", "@com_google_guava_guava", + "@mockito", "@truth", ], ) @@ -81,11 +83,11 @@ android_local_test( "//java/com/google/android/libraries/mobiledatadownload/file/openers:bytes", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//java/com/google/android/libraries/mobiledatadownload/internal/util:ProtoConversionUtil", + "//java/com/google/common/collect", "//javatests/com/google/android/libraries/mobiledatadownload/internal:MddTestUtil", "//proto:download_config_java_proto_lite", "//proto:transform_java_proto_lite", "@androidx_test", - "@com_google_guava_guava", "@com_google_protobuf//:parsers", "@com_google_protobuf//:protobuf_lite", "@com_google_testing//:test_util", diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/util/FuturesUtilTest.java b/javatests/com/google/android/libraries/mobiledatadownload/internal/util/FuturesUtilTest.java index e302f09..5a06251 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/internal/util/FuturesUtilTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/util/FuturesUtilTest.java @@ -30,9 +30,9 @@ import java.util.concurrent.Executors; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; +import org.robolectric.RobolectricTestRunner; -@RunWith(JUnit4.class) +@RunWith(RobolectricTestRunner.class) public final class FuturesUtilTest { private static final Executor SEQUENTIAL_EXECUTOR = diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/util/ProtoConversionUtilTest.java b/javatests/com/google/android/libraries/mobiledatadownload/internal/util/ProtoConversionUtilTest.java index 9dd366d..ad0a94b 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/internal/util/ProtoConversionUtilTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/util/ProtoConversionUtilTest.java @@ -20,6 +20,9 @@ import static com.google.common.truth.Truth.assertThat; import android.content.Context; import android.net.Uri; import androidx.test.core.app.ApplicationProvider; +import com.google.mobiledatadownload.internal.MetadataProto; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupBookkeeping; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; import com.google.android.libraries.mobiledatadownload.file.backends.AndroidFileBackend; import com.google.android.libraries.mobiledatadownload.file.backends.JavaFileBackend; @@ -40,9 +43,6 @@ import com.google.mobiledatadownload.TransformProto.CompressTransform; import com.google.mobiledatadownload.TransformProto.Transform; import com.google.mobiledatadownload.TransformProto.Transforms; import com.google.mobiledatadownload.TransformProto.ZipTransform; -import com.google.mobiledatadownload.internal.MetadataProto; -import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupBookkeeping; -import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; import com.google.protobuf.ExtensionRegistryLite; import com.google.protobuf.contrib.android.ProtoParsers; import com.google.testing.util.TestUtil; @@ -67,7 +67,7 @@ public final class ProtoConversionUtilTest { // The raw test data folder in google3. private static final String TEST_DATA_DIR = TestUtil.getRunfilesDir() - + "/google3/third_party/java_src/android_libs/mobiledatadownload/javatests/com/google/android/libraries/mobiledatadownload/internal/util/testdata/"; + + "/third_party/java_src/android_libs/mobiledatadownload/javatests/com/google/android/libraries/mobiledatadownload/internal/util/testdata/"; private static final File RAW_GROUP_WITH_EXTENSION = new File(TEST_DATA_DIR, "raw_group_with_extension"); @@ -146,7 +146,7 @@ public final class ProtoConversionUtilTest { public void convert_parseRawProtoWithExtensions() throws Exception { DataFileGroupInternal expected = ProtoConversionUtil.convert( - MddTestUtil.createDataFileGroup(/*fileGroupName=*/ "test-group", 2)) + MddTestUtil.createDataFileGroup(/* fileGroupName= */ "test-group", 2)) .toBuilder() .setBookkeeping( DataFileGroupBookkeeping.newBuilder() diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/util/testdata/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/internal/util/testdata/BUILD index 7b8b8eb..14cb9c9 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/internal/util/testdata/BUILD +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/util/testdata/BUILD @@ -14,6 +14,7 @@ load("//tools/build_rules/text_to_binary:def.bzl", "proto_data") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) diff --git a/javatests/com/google/android/libraries/mobiledatadownload/lite/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/lite/BUILD index d44327a..1dd50ba 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/lite/BUILD +++ b/javatests/com/google/android/libraries/mobiledatadownload/lite/BUILD @@ -14,6 +14,7 @@ load("//javatests/com/google/android/libraries/mobiledatadownload:test_defs.bzl", "mdd_local_test") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -43,12 +44,12 @@ mdd_local_test( deps = [ "//java/com/google/android/libraries/mobiledatadownload:DownloadException", "//java/com/google/android/libraries/mobiledatadownload/downloader:FileDownloader", + "//java/com/google/android/libraries/mobiledatadownload/foreground:ForegroundDownloadKey", "//java/com/google/android/libraries/mobiledatadownload/lite", "//java/com/google/android/libraries/mobiledatadownload/lite:DownloadListener", "//java/com/google/android/libraries/mobiledatadownload/lite:DownloadProgressMonitor", "//javatests/com/google/android/libraries/mobiledatadownload/testing:BlockingFileDownloader", "@android_sdk_linux", - "@androidx_test", "@com_google_guava_guava", "@mockito", "@truth", diff --git a/javatests/com/google/android/libraries/mobiledatadownload/lite/DownloaderImplTest.java b/javatests/com/google/android/libraries/mobiledatadownload/lite/DownloaderImplTest.java index abd2c07..2213da4 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/lite/DownloaderImplTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/lite/DownloaderImplTest.java @@ -31,6 +31,7 @@ import androidx.test.core.app.ApplicationProvider; import com.google.android.libraries.mobiledatadownload.DownloadException; import com.google.android.libraries.mobiledatadownload.downloader.DownloadConstraints; import com.google.android.libraries.mobiledatadownload.downloader.FileDownloader; +import com.google.android.libraries.mobiledatadownload.foreground.ForegroundDownloadKey; import com.google.android.libraries.mobiledatadownload.testing.BlockingFileDownloader; import com.google.common.base.Optional; import com.google.common.base.Supplier; @@ -76,6 +77,7 @@ public final class DownloaderImplTest { private Downloader downloader; private Context context; private DownloadRequest downloadRequest; + private ForegroundDownloadKey foregroundDownloadKey; private final Uri destinationFileUri = Uri.parse( "android://com.google.android.libraries.mobiledatadownload/files/datadownload/shared/public/file_1"); @@ -108,6 +110,8 @@ public final class DownloaderImplTest { .setNotificationContentTitle("File url: " + FILE_URL) .build(); + foregroundDownloadKey = ForegroundDownloadKey.ofSingleFile(destinationFileUri); + when(mockDownloadListener.onComplete()).thenReturn(Futures.immediateFuture(null)); } @@ -126,13 +130,13 @@ public final class DownloaderImplTest { Optional.of(mockDownloadMonitor), blockingDownloaderSupplier); - int downloadFuturesInFlightCountBefore = downloaderImpl.keyToListenableFuture.size(); + int downloadFuturesInFlightCountBefore = getInProgressFuturesCount(downloaderImpl); ListenableFuture<Void> downloadFuture1 = downloaderImpl.download(downloadRequest); ListenableFuture<Void> downloadFuture2 = downloaderImpl.download(downloadRequest); - assertThat(downloaderImpl.keyToListenableFuture).containsKey(destinationFileUri.toString()); - assertThat(downloaderImpl.keyToListenableFuture.size() - downloadFuturesInFlightCountBefore) + assertThat(containsInProgressFuture(downloaderImpl, foregroundDownloadKey.toString())).isTrue(); + assertThat(getInProgressFuturesCount(downloaderImpl) - downloadFuturesInFlightCountBefore) .isEqualTo(1); // Allow blocking download to finish @@ -144,12 +148,13 @@ public final class DownloaderImplTest { // TODO(b/155918406): Convert to Framework test and use TestingTaskBarrier to avoid sleep. // Sleep for 1 sec to wait for the Future's callback to finish. - Thread.sleep(/* millis = */ 1000); + Thread.sleep(/* millis= */ 1000); - // The completed download should be removed from keyToListenableFuture map. - assertThat(downloaderImpl.keyToListenableFuture) - .doesNotContainKey(destinationFileUri.toString()); - assertThat(downloaderImpl.keyToListenableFuture).hasSize(downloadFuturesInFlightCountBefore); + // The completed download should be removed from downloadFutureMap map. + assertThat(containsInProgressFuture(downloaderImpl, foregroundDownloadKey.toString())) + .isFalse(); + assertThat(getInProgressFuturesCount(downloaderImpl)) + .isEqualTo(downloadFuturesInFlightCountBefore); // Reset state of blockingFileDownloader to prevent deadlocks blockingFileDownloader.resetState(); @@ -170,14 +175,14 @@ public final class DownloaderImplTest { Optional.of(mockDownloadMonitor), blockingDownloaderSupplier); - int downloadFuturesInFlightCountBefore = downloaderImpl.keyToListenableFuture.size(); + int downloadFuturesInFlightCountBefore = getInProgressFuturesCount(downloaderImpl); ListenableFuture<Void> downloadFuture1 = downloaderImpl.downloadWithForegroundService(downloadRequest); ListenableFuture<Void> downloadFuture2 = downloaderImpl.download(downloadRequest); - assertThat(downloaderImpl.keyToListenableFuture).containsKey(destinationFileUri.toString()); - assertThat(downloaderImpl.keyToListenableFuture.size() - downloadFuturesInFlightCountBefore) + assertThat(containsInProgressFuture(downloaderImpl, foregroundDownloadKey.toString())).isTrue(); + assertThat(getInProgressFuturesCount(downloaderImpl) - downloadFuturesInFlightCountBefore) .isEqualTo(1); // Allow blocking download to finish @@ -189,12 +194,13 @@ public final class DownloaderImplTest { // TODO(b/155918406): Convert to Framework test and use TestingTaskBarrier to avoid sleep. // Sleep for 1 sec to wait for the Future's callback to finish. - Thread.sleep(/* millis = */ 1000); + Thread.sleep(/* millis= */ 1000); - // The completed download should be removed from keyToListenableFuture map. - assertThat(downloaderImpl.keyToListenableFuture) - .doesNotContainKey(destinationFileUri.toString()); - assertThat(downloaderImpl.keyToListenableFuture).hasSize(downloadFuturesInFlightCountBefore); + // The completed download should be removed from downloadFutureMap map. + assertThat(containsInProgressFuture(downloaderImpl, foregroundDownloadKey.toString())) + .isFalse(); + assertThat(getInProgressFuturesCount(downloaderImpl)) + .isEqualTo(downloadFuturesInFlightCountBefore); // Reset state of blockingFileDownloader to prevent deadlocks blockingFileDownloader.resetState(); @@ -226,7 +232,7 @@ public final class DownloaderImplTest { // TODO(b/155918406): Convert to Framework test and use TestingTaskBarrier to avoid sleep. // Sleep for 1 sec to wait for the Future's callback to finish. - Thread.sleep(/* millis = */ 1000); + Thread.sleep(/* millis= */ 1000); // Verify that correct DownloadRequest is sent to underlying FileDownloader com.google.android.libraries.mobiledatadownload.downloader.DownloadRequest @@ -287,11 +293,10 @@ public final class DownloaderImplTest { // TODO(b/155918406): Convert to Framework test and use TestingTaskBarrier to avoid sleep. // Sleep for 1 sec to wait for the Future's callback to finish. - Thread.sleep(/* millis = */ 1000); + Thread.sleep(/* millis= */ 1000); // Ensure that future is still removed from internal map - assertThat(downloaderImpl.keyToListenableFuture) - .doesNotContainKey(downloadRequest.destinationFileUri().toString()); + assertThat(containsInProgressFuture(downloaderImpl, destinationFileUri.toString())).isFalse(); // Verify that DownloadMonitor handled DownloadListener properly verify(mockDownloadMonitor).addDownloadListener(destinationFileUri, mockDownloadListener); @@ -342,8 +347,10 @@ public final class DownloaderImplTest { () -> { try { // Verify that future map still contains download future. - assertThat(downloaderImpl.keyToListenableFuture) - .containsKey(destinationFileUri.toString()); + assertThat( + containsInProgressFuture( + downloaderImpl, foregroundDownloadKey.toString())) + .isTrue(); blockingOnCompleteLatch.await(); } catch (InterruptedException e) { // Ignore. @@ -355,26 +362,27 @@ public final class DownloaderImplTest { })) .build(); - downloaderImpl.download(downloadRequest).get(); + ListenableFuture<Void> downloadFuture = downloaderImpl.download(downloadRequest); + downloadFuture.get(); // Verify that the download future map still contains the download future. - assertThat(downloaderImpl.keyToListenableFuture).containsKey(destinationFileUri.toString()); + assertThat(containsInProgressFuture(downloaderImpl, foregroundDownloadKey.toString())).isTrue(); // TODO(b/155918406): Convert to Framework test and use TestingTaskBarrier to avoid sleep. // Sleep for 1 sec to wait for the Future's callback to finish. - Thread.sleep(/* millis = */ 1000); + Thread.sleep(/* millis= */ 1000); // Finish the onComplete method. blockingOnCompleteLatch.countDown(); // TODO(b/155918406): Convert to Framework test and use TestingTaskBarrier to avoid sleep. // Sleep for 1 sec to wait for the Future's callback to finish. - Thread.sleep(/* millis = */ 1000); + Thread.sleep(/* millis= */ 1000); // The completed download should be removed from keyToListenableFuture map. - assertThat(downloaderImpl.keyToListenableFuture) - .doesNotContainKey(destinationFileUri.toString()); - assertThat(downloaderImpl.keyToListenableFuture).isEmpty(); + assertThat(containsInProgressFuture(downloaderImpl, foregroundDownloadKey.toString())) + .isFalse(); + assertThat(getInProgressFuturesCount(downloaderImpl)).isEqualTo(0); // Verify DownloadListener was added/removed verify(mockDownloadMonitor).addDownloadListener(eq(destinationFileUri), any()); @@ -443,14 +451,13 @@ public final class DownloaderImplTest { ListenableFuture<Void> downloadFuture = downloaderImpl.download(downloadRequest); - assertThat(downloaderImpl.keyToListenableFuture) - .containsKey(downloadRequest.destinationFileUri().toString()); + assertThat(containsInProgressFuture(downloaderImpl, foregroundDownloadKey.toString())).isTrue(); downloadFuture.cancel(true); // The download future should no longer be included in the future map - assertThat(downloaderImpl.keyToListenableFuture) - .doesNotContainKey(downloadRequest.destinationFileUri().toString()); + assertThat(containsInProgressFuture(downloaderImpl, foregroundDownloadKey.toString())) + .isFalse(); // Reset state of blocking file downloader to prevent deadlocks blockingFileDownloader.resetState(); @@ -491,8 +498,10 @@ public final class DownloaderImplTest { () -> { try { // Verify that future map still contains download future. - assertThat(downloaderImpl.keyToListenableFuture) - .containsKey(destinationFileUri.toString()); + assertThat( + containsInProgressFuture( + downloaderImpl, foregroundDownloadKey.toString())) + .isTrue(); blockingOnCompleteLatch.await(); } catch (InterruptedException e) { // Ignore. @@ -504,26 +513,26 @@ public final class DownloaderImplTest { })) .build(); - downloaderImpl.download(downloadRequest).get(); + ListenableFuture<Void> downloadFuture = downloaderImpl.download(downloadRequest); + downloadFuture.get(); // Verify that the download future map still contains the download future. - assertThat(downloaderImpl.keyToListenableFuture).containsKey(destinationFileUri.toString()); + assertThat(containsInProgressFuture(downloaderImpl, foregroundDownloadKey.toString())).isTrue(); // TODO(b/155918406): Convert to Framework test and use TestingTaskBarrier to avoid sleep. // Sleep for 1 sec to wait for the Future's callback to finish. - Thread.sleep(/* millis = */ 1000); + Thread.sleep(/* millis= */ 1000); // Finish the onComplete method. blockingOnCompleteLatch.countDown(); // TODO(b/155918406): Convert to Framework test and use TestingTaskBarrier to avoid sleep. // Sleep for 1 sec to wait for the Future's callback to finish. - Thread.sleep(/* millis = */ 1000); + Thread.sleep(/* millis= */ 1000); - // The completed download should be removed from keyToListenableFuture map. - assertThat(downloaderImpl.keyToListenableFuture) - .doesNotContainKey(destinationFileUri.toString()); - assertThat(downloaderImpl.keyToListenableFuture).isEmpty(); + // The completed download should be removed from download future map. + assertThat(containsInProgressFuture(downloaderImpl, destinationFileUri.toString())).isFalse(); + assertThat(getInProgressFuturesCount(downloaderImpl)).isEqualTo(0); } @Test @@ -630,16 +639,16 @@ public final class DownloaderImplTest { Optional.of(mockDownloadMonitor), blockingDownloaderSupplier); - int downloadFuturesInFlightCountBefore = downloaderImpl.keyToListenableFuture.size(); + int downloadFuturesInFlightCountBefore = getInProgressFuturesCount(downloaderImpl); ListenableFuture<Void> downloadFuture1 = downloaderImpl.downloadWithForegroundService(downloadRequest); ListenableFuture<Void> downloadFuture2 = downloaderImpl.downloadWithForegroundService(downloadRequest); - assertThat(downloaderImpl.keyToListenableFuture).containsKey(destinationFileUri.toString()); + assertThat(containsInProgressFuture(downloaderImpl, foregroundDownloadKey.toString())).isTrue(); - assertThat(downloaderImpl.keyToListenableFuture.size() - downloadFuturesInFlightCountBefore) + assertThat(getInProgressFuturesCount(downloaderImpl) - downloadFuturesInFlightCountBefore) .isEqualTo(1); // Now we let the 2 futures downloadFuture1 downloadFuture2 to run by opening the latch. @@ -651,11 +660,13 @@ public final class DownloaderImplTest { // TODO(b/147583059): Convert to Framework test and use TestingTaskBarrier to avoid sleep. // Sleep for 1 sec to wait for the Future's callback to finish. - Thread.sleep(/*millis=*/ 1000); - // The completed download is removed from the uriToListenableFuture Map. - assertThat(downloaderImpl.keyToListenableFuture) - .doesNotContainKey(destinationFileUri.toString()); - assertThat(downloaderImpl.keyToListenableFuture).hasSize(downloadFuturesInFlightCountBefore); + Thread.sleep(/* millis= */ 1000); + + // The completed download is removed from the download future Map. + assertThat(containsInProgressFuture(downloaderImpl, foregroundDownloadKey.toString())) + .isFalse(); + assertThat(getInProgressFuturesCount(downloaderImpl)) + .isEqualTo(downloadFuturesInFlightCountBefore); // Reset state of blockingFileDownloader to prevent deadlocks blockingFileDownloader.resetState(); @@ -677,15 +688,14 @@ public final class DownloaderImplTest { Optional.of(mockDownloadMonitor), blockingDownloaderSupplier); - int downloadFuturesInFlightCountBefore = downloaderImpl.keyToListenableFuture.size(); + int downloadFuturesInFlightCountBefore = getInProgressFuturesCount(downloaderImpl); ListenableFuture<Void> downloadFuture1 = downloaderImpl.download(downloadRequest); ListenableFuture<Void> downloadFuture2 = downloaderImpl.downloadWithForegroundService(downloadRequest); - assertThat(downloaderImpl.keyToListenableFuture).containsKey(destinationFileUri.toString()); - - assertThat(downloaderImpl.keyToListenableFuture.size() - downloadFuturesInFlightCountBefore) + assertThat(containsInProgressFuture(downloaderImpl, foregroundDownloadKey.toString())).isTrue(); + assertThat(getInProgressFuturesCount(downloaderImpl) - downloadFuturesInFlightCountBefore) .isEqualTo(1); // Now we let the 2 futures downloadFuture1 downloadFuture2 to run by opening the latch. @@ -697,11 +707,13 @@ public final class DownloaderImplTest { // TODO(b/155918406): Convert to Framework test and use TestingTaskBarrier to avoid sleep. // Sleep for 1 sec to wait for the Future's callback to finish. - Thread.sleep(/*millis=*/ 1000); - // The completed download is removed from the uriToListenableFuture Map. - assertThat(downloaderImpl.keyToListenableFuture) - .doesNotContainKey(destinationFileUri.toString()); - assertThat(downloaderImpl.keyToListenableFuture).hasSize(downloadFuturesInFlightCountBefore); + Thread.sleep(/* millis= */ 1000); + + // The completed download is removed from the download future Map. + assertThat(containsInProgressFuture(downloaderImpl, foregroundDownloadKey.toString())) + .isFalse(); + assertThat(getInProgressFuturesCount(downloaderImpl)) + .isEqualTo(downloadFuturesInFlightCountBefore); // Reset state of blockingFileDownloader to prevent deadlocks blockingFileDownloader.resetState(); @@ -733,7 +745,7 @@ public final class DownloaderImplTest { // TODO(b/147583059): Convert to Framework test and use TestingTaskBarrier to avoid sleep. // Sleep for 1 sec to wait for the Future's callback to finish. - Thread.sleep(/*millis=*/ 1000); + Thread.sleep(/* millis= */ 1000); // Verify that the correct DownloadRequest is sent to underderlying FileDownloader. com.google.android.libraries.mobiledatadownload.downloader.DownloadRequest @@ -797,7 +809,7 @@ public final class DownloaderImplTest { // TODO(b/147583059): Convert to Framework test and use TestingTaskBarrier to avoid sleep. // Sleep for 1 sec to wait for the Future's callback to finish. - Thread.sleep(/*millis=*/ 1000); + Thread.sleep(/* millis= */ 1000); // Verify that the correct DownloadRequest is sent to underderlying FileDownloader. com.google.android.libraries.mobiledatadownload.downloader.DownloadRequest @@ -861,13 +873,15 @@ public final class DownloaderImplTest { // Verify that before client's onComplete finishes, the on-going // download future map still contain this download. This means // the Foreground Download Service has not be shut down yet. - assertThat(downloaderImpl.keyToListenableFuture) - .containsKey(destinationFileUri.toString()); + assertThat( + containsInProgressFuture( + downloaderImpl, foregroundDownloadKey.toString())) + .isTrue(); blockingOnCompleteLatch.await(); } catch (InterruptedException e) { // Ignore. } - return Futures.immediateFuture(null); + return Futures.immediateVoidFuture(); }, BACKGROUND_EXECUTOR); } @@ -886,22 +900,22 @@ public final class DownloaderImplTest { // TODO(b/147583059): Convert to Framework test and use TestingTaskBarrier to avoid sleep. // Sleep for 1 sec to wait for the Future's callback to finish. - Thread.sleep(/*millis=*/ 1000); + Thread.sleep(/* millis= */ 1000); - // Verify that this download future has not been removed from the keyToListenableFuture map yet. - assertThat(downloaderImpl.keyToListenableFuture).containsKey(destinationFileUri.toString()); + // Verify that this download future has not been removed from the download future map yet. + assertThat(containsInProgressFuture(downloaderImpl, foregroundDownloadKey.toString())).isTrue(); // Now let's the onComplete finishes. blockingOnCompleteLatch.countDown(); // TODO(b/147583059): Convert to Framework test and use TestingTaskBarrier to avoid sleep. // Sleep for 1 sec to wait for the Future's callback on onComplete to finish. - Thread.sleep(/*millis=*/ 1000); + Thread.sleep(/* millis= */ 1000); - // The completed download is removed from the keyToListenableFuture Map. - assertThat(downloaderImpl.keyToListenableFuture) - .doesNotContainKey(destinationFileUri.toString()); - assertThat(downloaderImpl.keyToListenableFuture).isEmpty(); + // The completed download is removed from the download future Map. + assertThat(containsInProgressFuture(downloaderImpl, foregroundDownloadKey.toString())) + .isFalse(); + assertThat(getInProgressFuturesCount(downloaderImpl)).isEqualTo(0); verify(mockDownloadMonitor).removeDownloadListener(destinationFileUri); } @@ -938,7 +952,7 @@ public final class DownloaderImplTest { // TODO(b/147583059): Convert to Framework test and use TestingTaskBarrier to avoid sleep. // Sleep for 1 sec to wait for the listener to finish. - Thread.sleep(/*millis=*/ 1000); + Thread.sleep(/* millis= */ 1000); // Verify that the correct DownloadRequest is sent to underderlying FileDownloader. com.google.android.libraries.mobiledatadownload.downloader.DownloadRequest @@ -977,23 +991,23 @@ public final class DownloaderImplTest { Optional.of(mockDownloadMonitor), blockingDownloaderSupplier); - int downloadFuturesInFlightCountBefore = downloaderImpl.keyToListenableFuture.size(); + int downloadFuturesInFlightCountBefore = getInProgressFuturesCount(downloaderImpl); ListenableFuture<Void> downloadFuture = downloaderImpl.downloadWithForegroundService(downloadRequest); - assertThat(downloaderImpl.keyToListenableFuture).containsKey(destinationFileUri.toString()); - - assertThat(downloaderImpl.keyToListenableFuture.size() - downloadFuturesInFlightCountBefore) + assertThat(containsInProgressFuture(downloaderImpl, foregroundDownloadKey.toString())).isTrue(); + assertThat(getInProgressFuturesCount(downloaderImpl) - downloadFuturesInFlightCountBefore) .isEqualTo(1); - downloaderImpl.cancelForegroundDownload(destinationFileUri.toString()); + downloaderImpl.cancelForegroundDownload(foregroundDownloadKey.toString()); assertTrue(downloadFuture.isCancelled()); - // The completed download is removed from the uriToListenableFuture Map. - assertThat(downloaderImpl.keyToListenableFuture) - .doesNotContainKey(destinationFileUri.toString()); - assertThat(downloaderImpl.keyToListenableFuture).hasSize(downloadFuturesInFlightCountBefore); + // The completed download is removed from the download future Map. + assertThat(containsInProgressFuture(downloaderImpl, foregroundDownloadKey.toString())) + .isFalse(); + assertThat(getInProgressFuturesCount(downloaderImpl)) + .isEqualTo(downloadFuturesInFlightCountBefore); // Reset state of blockingFileDownloader to prevent deadlocks blockingFileDownloader.resetState(); @@ -1017,15 +1031,25 @@ public final class DownloaderImplTest { ListenableFuture<Void> downloadFuture = downloaderImpl.downloadWithForegroundService(downloadRequest); - assertThat(downloaderImpl.keyToListenableFuture).containsKey(destinationFileUri.toString()); + assertThat(containsInProgressFuture(downloaderImpl, foregroundDownloadKey.toString())).isTrue(); downloadFuture.cancel(true); // The completed download is removed from the uriToListenableFuture Map. - assertThat(downloaderImpl.keyToListenableFuture) - .doesNotContainKey(destinationFileUri.toString()); + assertThat(containsInProgressFuture(downloaderImpl, foregroundDownloadKey.toString())) + .isFalse(); // Reset state of blockingFileDownloader to prevent deadlocks blockingFileDownloader.resetState(); } + + private static int getInProgressFuturesCount(DownloaderImpl downloaderImpl) { + return downloaderImpl.downloadFutureMap.keyToDownloadFutureMap.size() + + downloaderImpl.foregroundDownloadFutureMap.keyToDownloadFutureMap.size(); + } + + private static boolean containsInProgressFuture(DownloaderImpl downloaderImpl, String key) { + return downloaderImpl.downloadFutureMap.keyToDownloadFutureMap.containsKey(key) + || downloaderImpl.foregroundDownloadFutureMap.keyToDownloadFutureMap.containsKey(key); + } } diff --git a/javatests/com/google/android/libraries/mobiledatadownload/monitor/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/monitor/BUILD index d84b6be..32dcc42 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/monitor/BUILD +++ b/javatests/com/google/android/libraries/mobiledatadownload/monitor/BUILD @@ -14,6 +14,7 @@ load("//javatests/com/google/android/libraries/mobiledatadownload:test_defs.bzl", "mdd_local_test") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -23,14 +24,18 @@ mdd_local_test( srcs = ["NetworkUsageMonitorTest.java"], test_class = "com.google.android.libraries.mobiledatadownload.monitor.NetworkUsageMonitorTest", deps = [ + "//java/com/google/android/libraries/mobiledatadownload/file", + "//java/com/google/android/libraries/mobiledatadownload/file/backends:file", "//java/com/google/android/libraries/mobiledatadownload/file/common/testing", "//java/com/google/android/libraries/mobiledatadownload/file/spi", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LoggingStateStore", - "//java/com/google/android/libraries/mobiledatadownload/internal/logging:NoOpLoggingState", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:SharedPreferencesLoggingState", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//java/com/google/android/libraries/mobiledatadownload/monitor:NetworkUsageMonitor", "//javatests/com/google/android/libraries/mobiledatadownload/testing:FakeTimeSource", + "//javatests/com/google/android/libraries/mobiledatadownload/testing:MddTestDependencies", "@android_sdk_linux", + "@com_google_guava_guava", "@robolectric", "@truth", ], diff --git a/javatests/com/google/android/libraries/mobiledatadownload/monitor/NetworkUsageMonitorTest.java b/javatests/com/google/android/libraries/mobiledatadownload/monitor/NetworkUsageMonitorTest.java index 86aadb4..34e5b25 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/monitor/NetworkUsageMonitorTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/monitor/NetworkUsageMonitorTest.java @@ -26,12 +26,16 @@ import android.net.NetworkInfo.DetailedState; import android.net.Uri; import android.os.Build; import androidx.test.core.app.ApplicationProvider; +import com.google.mobiledatadownload.internal.MetadataProto.FileGroupLoggingState; +import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; import com.google.android.libraries.mobiledatadownload.file.common.testing.TemporaryUri; import com.google.android.libraries.mobiledatadownload.file.spi.Monitor; import com.google.android.libraries.mobiledatadownload.internal.logging.LoggingStateStore; -import com.google.android.libraries.mobiledatadownload.internal.logging.NoOpLoggingState; import com.google.android.libraries.mobiledatadownload.testing.FakeTimeSource; -import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; +import com.google.android.libraries.mobiledatadownload.testing.MddTestDependencies; +import com.google.common.base.Optional; +import java.util.List; +import java.util.Random; import java.util.concurrent.Executor; import org.junit.Before; import org.junit.Rule; @@ -86,7 +90,9 @@ public class NetworkUsageMonitorTest { public void setUp() throws Exception { context = ApplicationProvider.getApplicationContext(); - loggingStateStore = new NoOpLoggingState(); + loggingStateStore = + MddTestDependencies.LoggingStateStoreImpl.SHARED_PREFERENCES.loggingStateStore( + context, Optional.absent(), new FakeTimeSource(), executor, new Random()); // TODO(b/177015303): use builder when available networkUsageMonitor = new NetworkUsageMonitor(context, clock); @@ -119,9 +125,9 @@ public class NetworkUsageMonitorTest { .setVariantId(VARIANT_ID_1) .build(); networkUsageMonitor.monitorUri( - uri1, groupKey1, BUILD_ID_1, VERSION_NUMBER_1, loggingStateStore); + uri1, groupKey1, BUILD_ID_1, VARIANT_ID_1, VERSION_NUMBER_1, loggingStateStore); networkUsageMonitor.monitorUri( - uri2, groupKey1, BUILD_ID_1, VERSION_NUMBER_1, loggingStateStore); + uri2, groupKey1, BUILD_ID_1, VARIANT_ID_1, VERSION_NUMBER_1, loggingStateStore); GroupKey groupKey2 = GroupKey.newBuilder() @@ -131,7 +137,7 @@ public class NetworkUsageMonitorTest { .build(); networkUsageMonitor.monitorUri( - uri3, groupKey2, BUILD_ID_2, VERSION_NUMBER_2, loggingStateStore); + uri3, groupKey2, BUILD_ID_2, VARIANT_ID_2, VERSION_NUMBER_2, loggingStateStore); Monitor.OutputMonitor outputMonitor1 = networkUsageMonitor.monitorWrite(uri1); Monitor.OutputMonitor outputMonitor2 = networkUsageMonitor.monitorWrite(uri2); @@ -172,6 +178,27 @@ public class NetworkUsageMonitorTest { outputMonitor3.close(); // await executors idle here if we switch from directExecutor... + + List<FileGroupLoggingState> allLoggingState = loggingStateStore.getAndResetAllDataUsage().get(); + + assertThat(allLoggingState) + .containsExactly( + FileGroupLoggingState.newBuilder() + .setGroupKey(groupKey1) + .setBuildId(BUILD_ID_1) + .setVariantId(VARIANT_ID_1) + .setFileGroupVersionNumber(VERSION_NUMBER_1) + .setCellularUsage(16 + 32) + .setWifiUsage(1 + 2 + 4) + .build(), + FileGroupLoggingState.newBuilder() + .setGroupKey(groupKey2) + .setBuildId(BUILD_ID_2) + .setVariantId(VARIANT_ID_2) + .setFileGroupVersionNumber(VERSION_NUMBER_2) + .setCellularUsage(64) + .setWifiUsage(8) + .build()); } @Test @@ -186,9 +213,9 @@ public class NetworkUsageMonitorTest { .setVariantId(VARIANT_ID_1) .build(); networkUsageMonitor.monitorUri( - uri1, groupKey1, BUILD_ID_1, VERSION_NUMBER_1, loggingStateStore); + uri1, groupKey1, BUILD_ID_1, VARIANT_ID_1, VERSION_NUMBER_1, loggingStateStore); networkUsageMonitor.monitorUri( - uri2, groupKey1, BUILD_ID_1, VERSION_NUMBER_1, loggingStateStore); + uri2, groupKey1, BUILD_ID_1, VARIANT_ID_1, VERSION_NUMBER_1, loggingStateStore); GroupKey groupKey2 = GroupKey.newBuilder() @@ -199,9 +226,9 @@ public class NetworkUsageMonitorTest { // This would update uri2 to belong to FileGroup v2. networkUsageMonitor.monitorUri( - uri2, groupKey2, BUILD_ID_2, VERSION_NUMBER_2, loggingStateStore); + uri2, groupKey2, BUILD_ID_2, VARIANT_ID_2, VERSION_NUMBER_2, loggingStateStore); networkUsageMonitor.monitorUri( - uri3, groupKey2, BUILD_ID_2, VERSION_NUMBER_2, loggingStateStore); + uri3, groupKey2, BUILD_ID_2, VARIANT_ID_2, VERSION_NUMBER_2, loggingStateStore); Monitor.OutputMonitor outputMonitor1 = networkUsageMonitor.monitorWrite(uri1); Monitor.OutputMonitor outputMonitor2 = networkUsageMonitor.monitorWrite(uri2); @@ -241,6 +268,27 @@ public class NetworkUsageMonitorTest { outputMonitor1.close(); outputMonitor2.close(); outputMonitor3.close(); + + List<FileGroupLoggingState> allLoggingState = loggingStateStore.getAndResetAllDataUsage().get(); + + assertThat(allLoggingState) + .containsExactly( + FileGroupLoggingState.newBuilder() + .setGroupKey(groupKey1) + .setBuildId(BUILD_ID_1) + .setVariantId(VARIANT_ID_1) + .setFileGroupVersionNumber(VERSION_NUMBER_1) + .setCellularUsage(16) + .setWifiUsage(1 + 2) + .build(), + FileGroupLoggingState.newBuilder() + .setGroupKey(groupKey2) + .setBuildId(BUILD_ID_2) + .setVariantId(VARIANT_ID_2) + .setFileGroupVersionNumber(VERSION_NUMBER_2) + .setCellularUsage(32 + 64) + .setWifiUsage(4 + 8) + .build()); } @Test @@ -255,7 +303,7 @@ public class NetworkUsageMonitorTest { .setVariantId(VARIANT_ID_1) .build(); networkUsageMonitor.monitorUri( - uri1, groupKey1, BUILD_ID_1, VERSION_NUMBER_1, loggingStateStore); + uri1, groupKey1, BUILD_ID_1, VARIANT_ID_1, VERSION_NUMBER_1, loggingStateStore); Monitor.OutputMonitor outputMonitor1 = networkUsageMonitor.monitorWrite(uri1); @@ -265,6 +313,16 @@ public class NetworkUsageMonitorTest { // Downloaded 1 bytes on WIFI for uri1 setNetworkConnectivityType(ConnectivityManager.TYPE_WIFI); outputMonitor1.bytesWritten(new byte[1], 0, 1); + assertThat(loggingStateStore.getAndResetAllDataUsage().get()) + .containsExactly( + FileGroupLoggingState.newBuilder() + .setGroupKey(groupKey1) + .setBuildId(BUILD_ID_1) + .setVariantId(VARIANT_ID_1) + .setFileGroupVersionNumber(VERSION_NUMBER_1) + .setCellularUsage(0) + .setWifiUsage(1) + .build()); // Advance the clock by < LOG_FREQUENCY_SECONDS clock.advance(1, MILLISECONDS); @@ -279,6 +337,18 @@ public class NetworkUsageMonitorTest { // Advance the clock by > LOG_FREQUENCY_SECONDS clock.advance(NetworkUsageMonitor.LOG_FREQUENCY_SECONDS + 1, SECONDS); outputMonitor1.bytesWritten(new byte[16], 0, 8); + + // All chunks were saved. + assertThat(loggingStateStore.getAndResetAllDataUsage().get()) + .containsExactly( + FileGroupLoggingState.newBuilder() + .setGroupKey(groupKey1) + .setBuildId(BUILD_ID_1) + .setVariantId(VARIANT_ID_1) + .setFileGroupVersionNumber(VERSION_NUMBER_1) + .setCellularUsage(0) + .setWifiUsage(2 + 4 + 8) + .build()); } @Test @@ -294,9 +364,9 @@ public class NetworkUsageMonitorTest { .setVariantId(VARIANT_ID_1) .build(); networkUsageMonitor.monitorUri( - uri1, groupKey1, BUILD_ID_1, VERSION_NUMBER_1, loggingStateStore); + uri1, groupKey1, BUILD_ID_1, VARIANT_ID_1, VERSION_NUMBER_1, loggingStateStore); networkUsageMonitor.monitorUri( - uri2, groupKey1, BUILD_ID_1, VERSION_NUMBER_1, loggingStateStore); + uri2, groupKey1, BUILD_ID_1, VARIANT_ID_1, VERSION_NUMBER_1, loggingStateStore); GroupKey groupKey2 = GroupKey.newBuilder() @@ -306,7 +376,7 @@ public class NetworkUsageMonitorTest { .build(); networkUsageMonitor.monitorUri( - uri3, groupKey2, BUILD_ID_2, VERSION_NUMBER_2, loggingStateStore); + uri3, groupKey2, BUILD_ID_2, VARIANT_ID_2, VERSION_NUMBER_2, loggingStateStore); Monitor.OutputMonitor outputMonitor1 = networkUsageMonitor.monitorWrite(uri1); Monitor.OutputMonitor outputMonitor2 = networkUsageMonitor.monitorAppend(uri2); @@ -347,6 +417,27 @@ public class NetworkUsageMonitorTest { outputMonitor3.close(); // await executors idle here if we switch from directExecutor... + + List<FileGroupLoggingState> allLoggingState = loggingStateStore.getAndResetAllDataUsage().get(); + + assertThat(allLoggingState) + .containsExactly( + FileGroupLoggingState.newBuilder() + .setGroupKey(groupKey1) + .setBuildId(BUILD_ID_1) + .setVariantId(VARIANT_ID_1) + .setFileGroupVersionNumber(VERSION_NUMBER_1) + .setCellularUsage(16 + 32) + .setWifiUsage(1 + 2 + 4) + .build(), + FileGroupLoggingState.newBuilder() + .setGroupKey(groupKey2) + .setBuildId(BUILD_ID_2) + .setVariantId(VARIANT_ID_2) + .setFileGroupVersionNumber(VERSION_NUMBER_2) + .setCellularUsage(64) + .setWifiUsage(8) + .build()); } @Test diff --git a/javatests/com/google/android/libraries/mobiledatadownload/populator/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/populator/BUILD index 34ae724..0328438 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/populator/BUILD +++ b/javatests/com/google/android/libraries/mobiledatadownload/populator/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_local_test") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -43,7 +44,6 @@ android_local_test( }, deps = [ "//java/com/google/android/libraries/mobiledatadownload/populator:LocationProvider", - "@androidx_test", "@mockito", "@truth", ], @@ -91,6 +91,7 @@ android_local_test( "targetSdkVersion": "27", }, deps = [ + "//java/com/google/android/libraries/mobiledatadownload:Flags", "//java/com/google/android/libraries/mobiledatadownload/populator:LocaleOverrider", "//java/com/google/android/libraries/mobiledatadownload/populator:MigrationProxyLocaleOverrider", "//proto:download_config_java_proto_lite", diff --git a/javatests/com/google/android/libraries/mobiledatadownload/populator/ManifestConfigFlagPopulatorTest.java b/javatests/com/google/android/libraries/mobiledatadownload/populator/ManifestConfigFlagPopulatorTest.java index 1d4a9be..d0052e5 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/populator/ManifestConfigFlagPopulatorTest.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/populator/ManifestConfigFlagPopulatorTest.java @@ -26,6 +26,7 @@ import static org.mockito.Mockito.when; import com.google.android.libraries.mobiledatadownload.AddFileGroupRequest; import com.google.android.libraries.mobiledatadownload.MobileDataDownload; import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -89,7 +90,11 @@ public class ManifestConfigFlagPopulatorTest { ManifestConfig manifestConfig = createManifestConfig(); ManifestConfigHelper.refreshFromManifestConfig( - mockMobileDataDownload, manifestConfig, /*overriderOptional=*/ Optional.absent()) + mockMobileDataDownload, + manifestConfig, + /* overriderOptional= */ Optional.absent(), + /* accounts= */ ImmutableList.of(), + /* addGroupsWithVariantId= */ false) .get(); verify(mockMobileDataDownload, times(2)).addFileGroup(addFileGroupRequestCaptor.capture()); @@ -120,7 +125,9 @@ public class ManifestConfigFlagPopulatorTest { ManifestConfigHelper.refreshFromManifestConfig( mockMobileDataDownload, manifestConfigWithUrlTemplate, - /*overriderOptional=*/ Optional.absent()) + /* overriderOptional= */ Optional.absent(), + /* accounts= */ ImmutableList.of(), + /* addGroupsWithVariantId= */ false) .get(); verify(mockMobileDataDownload, times(2)).addFileGroup(addFileGroupRequestCaptor.capture()); @@ -144,7 +151,9 @@ public class ManifestConfigFlagPopulatorTest { ManifestConfigHelper.refreshFromManifestConfig( mockMobileDataDownload, manifestConfigWithoutUrlTemplate, - /*overriderOptional=*/ Optional.absent()) + /* overriderOptional= */ Optional.absent(), + /* accounts= */ ImmutableList.of(), + /* addGroupsWithVariantId= */ false) .get()); assertThat(illegalArgumentException) @@ -172,7 +181,11 @@ public class ManifestConfigFlagPopulatorTest { }; ManifestConfigHelper.refreshFromManifestConfig( - mockMobileDataDownload, manifestConfig, Optional.of(overrider)) + mockMobileDataDownload, + manifestConfig, + Optional.of(overrider), + /* accounts= */ ImmutableList.of(), + /* addGroupsWithVariantId= */ false) .get(); verify(mockMobileDataDownload, times(2)).addFileGroup(addFileGroupRequestCaptor.capture()); @@ -204,7 +217,11 @@ public class ManifestConfigFlagPopulatorTest { }; ManifestConfigHelper.refreshFromManifestConfig( - mockMobileDataDownload, manifestConfig, Optional.of(overrider)) + mockMobileDataDownload, + manifestConfig, + Optional.of(overrider), + /* accounts= */ ImmutableList.of(), + /* addGroupsWithVariantId= */ false) .get(); verify(mockMobileDataDownload, times(1)).addFileGroup(addFileGroupRequestCaptor.capture()); diff --git a/javatests/com/google/android/libraries/mobiledatadownload/test_defs.bzl b/javatests/com/google/android/libraries/mobiledatadownload/test_defs.bzl index 964756d..3268a1b 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/test_defs.bzl +++ b/javatests/com/google/android/libraries/mobiledatadownload/test_defs.bzl @@ -58,6 +58,7 @@ register_extension_info( _EMULATOR_IMAGES = [ # Automotive "//tools/android/emulated_devices/automotive:auto_29_x86", + "//tools/android/emulated_devices/automotive:auto_30_x86", # Android Phone "//tools/android/emulated_devices/generic_phone:google_21_x86_gms_stable", @@ -80,6 +81,23 @@ _COMMON_LOGCAT_ARGS = [ # This is a workaround for b/111061456. _EMPTY_LOCAL_RESOURCE_FILES = [] +# Parameterized Integration Tests use TestParameterInjector (only supported at API level >= 24) +# This list represents the emulator images that should be used rather than the default full list. +PARAMETERIZED_EMULATOR_IMAGES = [ + # Automotive + "//tools/android/emulated_devices/automotive:auto_29_x86", + "//tools/android/emulated_devices/automotive:auto_30_x86", + + # Android Phone + "//tools/android/emulated_devices/generic_phone:google_24_x86_gms_stable", + "//tools/android/emulated_devices/generic_phone:google_25_x86_gms_stable", + "//tools/android/emulated_devices/generic_phone:google_26_x86_gms_stable", + "//tools/android/emulated_devices/generic_phone:google_27_x86_gms_stable", + "//tools/android/emulated_devices/generic_phone:google_28_x86_gms_stable", + "//tools/android/emulated_devices/generic_phone:google_29_x86_gms_stable", + "//tools/android/emulated_devices/generic_phone:google_30_x86_gms_stable", +] + # Wrapper around android_application_test to generate targets for multiple emulator images. def mdd_android_test(name, target_devices = _EMULATOR_IMAGES, **kwargs): """Generate an android_application_test for MDD. @@ -133,23 +151,23 @@ def mdd_android_test(name, target_devices = _EMULATOR_IMAGES, **kwargs): ) # Wrapper around check_dependencies. -def dependencies_test(name, allowlist = [], **kwargs): +def dependencies_test(name, whitelist = [], **kwargs): """Generate a dependencies_test for MDD. Args: name: The test name. - allowlist: The excluded targets under the package. + whitelist: The excluded targets under the package. **kwargs: Any keyword arguments to be passed. """ all_builds = [] for r in native.existing_rules().values(): - allowlisted = False - for build in allowlist: + whitelisted = False + for build in whitelist: # Ignore the leading colon in build. if build[1:] in r["name"]: - allowlisted = True + whitelisted = True break - if not allowlisted: + if not whitelisted: all_builds.append(r["name"]) check_dependencies( name = name, diff --git a/javatests/com/google/android/libraries/mobiledatadownload/testdata/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/testdata/BUILD index dec506c..0aaa728 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/testdata/BUILD +++ b/javatests/com/google/android/libraries/mobiledatadownload/testdata/BUILD @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -20,7 +21,7 @@ filegroup( name = "integration_test_data_files", testonly = 1, srcs = [ - "odws1_empty", + "odws1_empty.jar", "step1.txt", "step2.txt", "zip_test_folder.zip", @@ -32,6 +33,7 @@ filegroup( testonly = 1, srcs = [ "full_file.txt", + "full_file.zlib", "partial_file.txt", ], ) diff --git a/javatests/com/google/android/libraries/mobiledatadownload/testdata/full_file.zlib b/javatests/com/google/android/libraries/mobiledatadownload/testdata/full_file.zlib Binary files differnew file mode 100644 index 0000000..db0c61d --- /dev/null +++ b/javatests/com/google/android/libraries/mobiledatadownload/testdata/full_file.zlib diff --git a/javatests/com/google/android/libraries/mobiledatadownload/testdata/odws1_empty.jar b/javatests/com/google/android/libraries/mobiledatadownload/testdata/odws1_empty.jar Binary files differnew file mode 100644 index 0000000..1c990c4 --- /dev/null +++ b/javatests/com/google/android/libraries/mobiledatadownload/testdata/odws1_empty.jar diff --git a/javatests/com/google/android/libraries/mobiledatadownload/testdata/subpackage/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/testdata/subpackage/BUILD new file mode 100644 index 0000000..3235d52 --- /dev/null +++ b/javatests/com/google/android/libraries/mobiledatadownload/testdata/subpackage/BUILD @@ -0,0 +1,26 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//:__subpackages__"], + licenses = ["notice"], +) + +filegroup( + name = "multi_directory_downloader_test_data_files", + testonly = 1, + srcs = [ + "step3.txt", + ], +) diff --git a/javatests/com/google/android/libraries/mobiledatadownload/testdata/subpackage/step3.txt b/javatests/com/google/android/libraries/mobiledatadownload/testdata/subpackage/step3.txt new file mode 100644 index 0000000..7b60bbb --- /dev/null +++ b/javatests/com/google/android/libraries/mobiledatadownload/testdata/subpackage/step3.txt @@ -0,0 +1 @@ +step3.txt
\ No newline at end of file diff --git a/javatests/com/google/android/libraries/mobiledatadownload/testing/AndroidManifest.xml b/javatests/com/google/android/libraries/mobiledatadownload/testing/AndroidManifest.xml index a06e4b6..8379a23 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/testing/AndroidManifest.xml +++ b/javatests/com/google/android/libraries/mobiledatadownload/testing/AndroidManifest.xml @@ -17,9 +17,13 @@ --> <?xml version="1.0" encoding="utf-8"?> <manifest xmlns:android="http://schemas.android.com/apk/res/android" + xmlns:tools="http://schemas.android.com/tools" package="com.google.android.libraries.mobiledatadownload.testing" > - <uses-sdk android:minSdkVersion="16" /> +<!-- Use min sdk of 16, but allow TestParameterInjector to override this since its min sdk is 24 --> +<uses-sdk + tools:overrideLibrary="com.google.android.libraries.mobiledatadownload.testing, com.google.testing.junit.testparameterinjector" + android:minSdkVersion="16" /> <uses-permission android:name="android.permission.INTERNET"/> <uses-permission diff --git a/javatests/com/google/android/libraries/mobiledatadownload/testing/BUILD b/javatests/com/google/android/libraries/mobiledatadownload/testing/BUILD index 94c4af0..8359f40 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/testing/BUILD +++ b/javatests/com/google/android/libraries/mobiledatadownload/testing/BUILD @@ -14,10 +14,51 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) +package_group( + name = "visibility_group", + packages = [ + "//java/com/google/android/apps/search/assistant/verticals/ambient/places/hammerdb/testing/...", + "//java/com/google/android/apps/tycho/common/download/largefile/testing/...", + "//java/com/google/android/libraries/lens/view/download/...", + "//java/com/google/android/libraries/translate/...", + "//javatests/com/google/android/apps/gsa/shared/speech/hotword/...", + "//javatests/com/google/android/apps/gsa/staticplugins/mdd/...", + "//javatests/com/google/android/apps/inputmethod/...", + "//javatests/com/google/android/apps/search/assistant/platform/ondevice/datadownload/...", + "//javatests/com/google/android/apps/search/assistant/surfaces/voice/initialdownload/...", + "//javatests/com/google/android/apps/search/assistant/verticals/ambient/places/hammerdb/...", + "//javatests/com/google/android/apps/search/assistant/verticals/ambient/places/shared/...", + "//javatests/com/google/android/apps/search/assistant/verticals/ambient/places/slices/...", + "//javatests/com/google/android/apps/search/fedora/...", + "//javatests/com/google/android/apps/translate/...", + "//javatests/com/google/android/apps/turbo/...", + "//javatests/com/google/android/apps/tycho/common/download/largefile/...", + "//javatests/com/google/android/apps/youtube/app/common/devicecapabilities/...", + "//javatests/com/google/android/gmscore/integ/modules/userprofile/...", + "//javatests/com/google/android/libraries/assistant/...", + "//javatests/com/google/android/libraries/compose/...", + "//javatests/com/google/android/libraries/inputmethod/...", + "//javatests/com/google/android/libraries/lens/view/...", + "//javatests/com/google/android/libraries/lens/view/download/...", + "//javatests/com/google/android/libraries/mobiledatadownload/file/...", + "//javatests/com/google/android/libraries/platformcommunications/expressiondata/...", + "//javatests/com/google/android/libraries/search/integrations/mdd/...", + "//javatests/com/google/android/libraries/search/soda/resourcemanager/...", + "//javatests/com/google/android/libraries/speech/modeldownload/contextual/...", + "//javatests/com/google/android/libraries/translate/...", + "//javatests/com/google/android/libraries/youtube/innertube/datapush/...", + "//javatests/com/google/android/libraries/youtube/studio/commands/...", + "//third_party/java_src/android_app/bugle/shared/java/com/google/android/apps/messaging/shared/mdd/testing", + "//third_party/java_src/android_app/bugle/tests/robolectric/javatests/com/google/android/apps/messaging/shared/mdd/...", + "//third_party/java_src/android_app/dialer/java/com/android/dialer/mobiledatadownload/testing", + ], +) + android_library( name = "BlockingFileDownloader", testonly = 1, @@ -48,6 +89,7 @@ android_library( srcs = ["FakeTimeSource.java"], deps = [ "//java/com/google/android/libraries/mobiledatadownload:TimeSource", + "@com_google_errorprone_error_prone_annotations", ], ) @@ -122,7 +164,57 @@ android_library( srcs = ["TestHttpServer.java"], deps = [ "@android_sdk_linux", + "@com_google_errorprone_error_prone_annotations", + "@com_google_guava_guava", + ], +) + +android_library( + name = "MddTestDependencies", + testonly = 1, + srcs = ["MddTestDependencies.java"], + deps = [ + "//java/com/google/android/libraries/mobiledatadownload:Flags", + "//java/com/google/android/libraries/mobiledatadownload:TimeSource", + "//java/com/google/android/libraries/mobiledatadownload/downloader:FileDownloader", + "//java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2:base", + "//java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2:base_deps", + "//java/com/google/android/libraries/mobiledatadownload/file", + "//java/com/google/android/libraries/mobiledatadownload/file/backends:android", + "//java/com/google/android/libraries/mobiledatadownload/file/integration/downloader:downloader2_sp", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LoggingStateStore", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:SharedPreferencesLoggingState", + "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "//java/com/google/android/libraries/mobiledatadownload/monitor:DownloadProgressMonitor", "@com_google_guava_guava", + "@cronet-api", + ], +) + +android_library( + name = "FakeMobileDataDownload", + testonly = 1, + srcs = [ + "FakeMobileDataDownload.java", + ], + deps = [ + "//java/com/google/android/libraries/mobiledatadownload", + "//java/com/google/android/libraries/mobiledatadownload:DownloadException", + "//java/com/google/android/libraries/mobiledatadownload:UsageEvent", + "//java/com/google/android/libraries/mobiledatadownload/account:AccountUtil", + "//java/com/google/android/libraries/mobiledatadownload/file", + "//java/com/google/android/libraries/mobiledatadownload/file/backends:android", + "//java/com/google/android/libraries/mobiledatadownload/file/openers:stream", + "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "//proto:client_config_java_proto_lite", + "//proto:download_config_java_proto_lite", + "@androidx_test", + "@com_google_code_findbugs_jsr305", + "@com_google_dagger", + "@com_google_guava_guava", + "@flogger", + "@javax_inject", ], ) diff --git a/javatests/com/google/android/libraries/mobiledatadownload/testing/FakeMobileDataDownload.java b/javatests/com/google/android/libraries/mobiledatadownload/testing/FakeMobileDataDownload.java new file mode 100644 index 0000000..3882756 --- /dev/null +++ b/javatests/com/google/android/libraries/mobiledatadownload/testing/FakeMobileDataDownload.java @@ -0,0 +1,640 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.testing; + +import android.accounts.Account; +import android.net.Uri; +import androidx.test.core.app.ApplicationProvider; +import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; +import com.google.android.libraries.mobiledatadownload.AddFileGroupRequest; +import com.google.android.libraries.mobiledatadownload.DownloadException; +import com.google.android.libraries.mobiledatadownload.DownloadException.DownloadResultCode; +import com.google.android.libraries.mobiledatadownload.DownloadFileGroupRequest; +import com.google.android.libraries.mobiledatadownload.GetFileGroupRequest; +import com.google.android.libraries.mobiledatadownload.GetFileGroupsByFilterRequest; +import com.google.android.libraries.mobiledatadownload.ImportFilesRequest; +import com.google.android.libraries.mobiledatadownload.MobileDataDownload; +import com.google.android.libraries.mobiledatadownload.ReadDataFileGroupRequest; +import com.google.android.libraries.mobiledatadownload.RemoveFileGroupRequest; +import com.google.android.libraries.mobiledatadownload.RemoveFileGroupsByFilterRequest; +import com.google.android.libraries.mobiledatadownload.RemoveFileGroupsByFilterResponse; +import com.google.android.libraries.mobiledatadownload.SingleFileDownloadRequest; +import com.google.android.libraries.mobiledatadownload.TaskScheduler.ConstraintOverrides; +import com.google.android.libraries.mobiledatadownload.UsageEvent; +import com.google.android.libraries.mobiledatadownload.account.AccountUtil; +import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; +import com.google.android.libraries.mobiledatadownload.file.backends.AndroidUri; +import com.google.android.libraries.mobiledatadownload.file.openers.WriteStreamOpener; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; +import com.google.common.base.Optional; +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.Table; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.mobiledatadownload.ClientConfigProto.ClientFile; +import com.google.mobiledatadownload.ClientConfigProto.ClientFileGroup; +import com.google.mobiledatadownload.DownloadConfigProto.DataFile; +import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executor; + +/** + * Fake implementation of {@link MobileDataDownload}. + * + * <p>FakeMobileDataDownload is thread-safe. All the apis part of MobileDataDownload interface can + * be invoked from multiple threads safely. Thread safety for helper functions (like setUpFileGroup, + * setThrowable, setThrowableOnFileGroup, get*Params apis etc) is not provided. To avoid race + * conditions, all the set up functions should be invoked at the beginning of the test before + * testing the business logic and get*Params apis should be invoked only after all the pending tasks + * are done. Refer <internal> to wait for all the pending background asynchronous tasks to complete. + */ +public final class FakeMobileDataDownload implements MobileDataDownload { + +// private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); + + private final List<AddFileGroupRequest> addFileGroupParamsList = new ArrayList<>(); + private final List<ClientFileGroup> downloadedFileGroupList = new ArrayList<>(); + private final List<GetFileGroupRequest> getFileGroupParamsList = new ArrayList<>(); + private final List<String> handleTaskParamsList = new ArrayList<>(); + private final List<ClientFileGroup> pendingFileGroupList = new ArrayList<>(); + private final Map<MethodType, Throwable> throwableMap = new EnumMap<>(MethodType.class); + private final Table<MethodType, GroupKey, Throwable> methodTypeGroupKeyToThrowableTable = + HashBasedTable.create(); + private final List<DownloadFileGroupRequest> downloadFileGroupParamsList = new ArrayList<>(); + private final List<DownloadFileGroupRequest> downloadFileGroupWithForegroundServiceParamsList = + new ArrayList<>(); + private final List<RemoveFileGroupRequest> removeFileGroupParamsList = new ArrayList<>(); + private final Map<String, byte[]> remoteFilesMap = new HashMap<>(); + + private final Optional<SynchronousFileStorage> storageOptional; + private final Executor sequentialControlExecutor; + + /** Enum for different MDD methods. Used to set Throwable. */ + public enum MethodType { + ADD_FILE_GROUP, + GET_FILE_GROUP, + REMOVE_FILE_GROUP, + DOWNLOAD_FILE, + DOWNLOAD_FILE_FOREGROUND, + } + + /** {@code storageOptional} must be present to download files set through setUpRemoteFile. */ + FakeMobileDataDownload(Optional<SynchronousFileStorage> storageOptional, Executor executor) { + this.storageOptional = storageOptional; + this.sequentialControlExecutor = MoreExecutors.newSequentialExecutor(executor); + } + + public static FakeMobileDataDownload createFakeMddWithFileStorage( + SynchronousFileStorage storage) { + return new FakeMobileDataDownload( + Optional.of(storage), MoreExecutors.newSequentialExecutor(MoreExecutors.directExecutor())); + } + + private static List<ClientFileGroup> getMatchingFileGroups( + GroupKey groupKey, List<ClientFileGroup> fileGroupList) { +// logger.atConfig().log("#getMatchingFileGroups: %s, %s", groupKey, fileGroupList); + List<ClientFileGroup> filteredFileGroupList = new ArrayList<>(); + for (ClientFileGroup fileGroup : fileGroupList) { + // Check for group name match. + if (groupKey.hasGroupName() && !groupKey.getGroupName().equals(fileGroup.getGroupName())) { + continue; + } + + // Check for owner_package match. + if (groupKey.hasOwnerPackage() + && !groupKey.getOwnerPackage().equals(fileGroup.getOwnerPackage())) { + continue; + } + + // Check for account match. + if (groupKey.hasAccount() && !groupKey.getAccount().equals(fileGroup.getAccount())) { + continue; + } + + // Check for variant id match. + if (groupKey.hasVariantId() && !groupKey.getVariantId().equals(fileGroup.getVariantId())) { + continue; + } + + filteredFileGroupList.add(fileGroup); + } + + return filteredFileGroupList; + } + + /** + * Sets {@link ClientFileGroup} instance to use in getFileGroup, getFileGroupsByFilter and + * downloadFileGroup methods. + * + * <p>getFileGroup, getFileGroupsByFilter, downloadFileGroup methods will look for a match in all + * the file groups set using this api before returning the result. + * + * @param clientFileGroup ClientFileGroup instance. + * @param downloaded if true, assumes the ClientFileGroup instance is downloaded, else download is + * pending. + */ + public void setUpFileGroup(ClientFileGroup clientFileGroup, boolean downloaded) { + if (downloaded) { + downloadedFileGroupList.add( + clientFileGroup.toBuilder().setStatus(ClientFileGroup.Status.DOWNLOADED).build()); + } else { + pendingFileGroupList.add( + clientFileGroup.toBuilder().setStatus(ClientFileGroup.Status.PENDING).build()); + } + } + + /** + * Returns the list of parameters that addFileGroup method was invocated with. + * + * @return List of all the requests of type {@link AddFileGroupRequest} that addFileGroup method + * was called with. + */ + public ImmutableList<AddFileGroupRequest> getAddFileGroupParamsList() { + return ImmutableList.copyOf(addFileGroupParamsList); + } + + /** + * Returns the list of parameters that removeFileGroup method was invocated with. + * + * @return List of all the requests of type {@link RemoveFileGroupRequest} that removeFileGroup + * method was called with. + */ + public ImmutableList<RemoveFileGroupRequest> getRemoveFileGroupParamsList() { + return ImmutableList.copyOf(removeFileGroupParamsList); + } + + /** + * Returns the list of parameters that downloadFileGroup method was invocated with. + * + * @return List of all the requests of type {@link DownloadFileGroupRequest} that + * downloadFileGroup method was called with. + */ + public ImmutableList<DownloadFileGroupRequest> getDownloadFileGroupParamsList() { + return ImmutableList.copyOf(downloadFileGroupParamsList); + } + + /** + * Returns the list of parameters that downloadFileGroupWithForegroundService method was invocated + * with. + * + * @return List of all the requests of type {@link DownloadFileGroupRequest} that + * downloadFileGroup method was called with. + */ + public ImmutableList<DownloadFileGroupRequest> + getDownloadFileGroupWithForegroundServiceParamsList() { + return ImmutableList.copyOf(downloadFileGroupWithForegroundServiceParamsList); + } + + /** + * Returns the list of parameters that getFileGroup method was invocated with. + * + * @return List of all the requests of type {@link GetFileGroupRequest} that getFileGroup method + * was called with. + */ + public ImmutableList<GetFileGroupRequest> getGetFileGroupParamsList() { + return ImmutableList.copyOf(getFileGroupParamsList); + } + + /** Returns the list of parameters that handleTask method was invocated with. */ + public ImmutableList<String> getHandleTaskParamsList() { + return ImmutableList.copyOf(handleTaskParamsList); + } + + /** + * Sets {@code throwable} to throw on invocation of a method identified by {@code methodType} + * + * @param methodType enum to identify method. + * @param throwable Throwable to throw on method's invocation. + */ + public void setThrowable(MethodType methodType, Throwable throwable) { + this.throwableMap.put(methodType, throwable); + } + + /** + * Sets {@code throwable} to throw on invocation of method identified by {@code methodType} when + * the properties set using {@code groupName}, {@code variantIdOptional}, {@code accountOptional} + * matches with the filegroup on which the method is invoked. + * + * @param methodType enum to identify method. + * @param groupName Name of the file group. + * @param accountOptional Account of the file group. Setting this is optional. + * @param variantIdOptional Variant Id of the file group. Setting this is optional. + * @param throwable Throwable to throw. + * <p>If throwable is set using both #setThrowable and #setThrowableOnFileGroup for a method, + * priority is given to throwable set through the latter. + */ + public void setThrowableOnFileGroup( + MethodType methodType, + String groupName, + Optional<Account> accountOptional, + Optional<String> variantIdOptional, + Throwable throwable) { + if (methodType != MethodType.GET_FILE_GROUP) { + throw new IllegalArgumentException( + "setThrowableOnFileGroup is currently only supported for getFileGroup method."); + } + GroupKey.Builder groupKeyBuilder = GroupKey.newBuilder(); + groupKeyBuilder.setGroupName(groupName); + if (accountOptional.isPresent()) { + groupKeyBuilder.setAccount(AccountUtil.serialize(accountOptional.get())); + } + if (variantIdOptional.isPresent()) { + groupKeyBuilder.setVariantId(variantIdOptional.get()); + } + methodTypeGroupKeyToThrowableTable.put(methodType, groupKeyBuilder.build(), throwable); + } + + /** + * Set file corresponding to a url. + * + * <p>Used by downloadFile and downloadFileWithForegroundService. If the + * SingleFileDownloadRequest#urlToDownload matches any of the set url, file is created at + * SingleFileDownloadRequest#destinationFileUri with the corresponding set content. + * + * <p>Setting content for an already existing url will replace the existing contents. + */ + public void setUpRemoteFile(String urlToDownload, byte[] content) { + // NOTE: If a client is using AssetFileBackend, then the corresponding test assets can be + // used here if the parameter type is Uri instead of byte[]. + // Note: Here byte[] will be stored in memory. Uri avoids this and supports large files cleanly. + remoteFilesMap.put(urlToDownload, content); + } + + @Override + public ListenableFuture<Boolean> addFileGroup(AddFileGroupRequest addFileGroupRequest) { +// logger.atInfo().log("#addFileGroup: %s", addFileGroupRequest); + Throwable addFileGroupThrowable = throwableMap.get(MethodType.ADD_FILE_GROUP); + if (addFileGroupThrowable != null) { + return Futures.immediateFailedFuture(addFileGroupThrowable); + } + addFileGroupParamsList.add(addFileGroupRequest); + + // Let addFileGroup induce realistic behavior. + // Wrap in background executor because this might do disk reads. + return PropagatedFutures.submitAsync( + () -> { + setUpFileGroup(toClientFileGroup(addFileGroupRequest), false); + return Futures.immediateFuture(true); + }, + sequentialControlExecutor); + } + + private ClientFileGroup toClientFileGroup(AddFileGroupRequest addFileGroupRequest) { + ClientFileGroup.Builder clientFileGroupBuilder = + ClientFileGroup.newBuilder() + .setGroupName(addFileGroupRequest.dataFileGroup().getGroupName()); + if (addFileGroupRequest.accountOptional().isPresent()) { + clientFileGroupBuilder.setAccount( + AccountUtil.serialize(addFileGroupRequest.accountOptional().get())); + } + if (addFileGroupRequest.dataFileGroup().hasOwnerPackage()) { + clientFileGroupBuilder.setOwnerPackage(addFileGroupRequest.dataFileGroup().getOwnerPackage()); + } + if (addFileGroupRequest.variantIdOptional().isPresent()) { + clientFileGroupBuilder.setVariantId(addFileGroupRequest.variantIdOptional().get()); + } + for (DataFile dataFile : addFileGroupRequest.dataFileGroup().getFileList()) { + ClientFile.Builder clientFileBuilder = + ClientFile.newBuilder().setFileId(dataFile.getFileId()); + if (dataFile.hasUrlToDownload()) { + String urlToDownload = dataFile.getUrlToDownload(); + clientFileBuilder.setFileUri(getMobstoreUriForRemoteFile(urlToDownload).toString()); + maybeSetUpFileAtUri(urlToDownload); + } + clientFileGroupBuilder.addFile(clientFileBuilder); + } + + return clientFileGroupBuilder.build(); + } + + private void maybeSetUpFileAtUri(String urlToDownload) { + if (storageOptional.isPresent() && remoteFilesMap.containsKey(urlToDownload)) { + try { + Uri mobstoreUri = getMobstoreUriForRemoteFile(urlToDownload); + storageOptional + .get() + .open(mobstoreUri, WriteStreamOpener.create()) + .write(remoteFilesMap.get(urlToDownload)); +// logger.atInfo().log( +// "Writing file for URL %s to Mobstore URI: %s", urlToDownload, mobstoreUri); + } catch (IOException e) { +// logger.atSevere().withCause(e).log("Mobstore file write failed"); + } + } else { +// logger.atConfig().log( +// "No file set for %s. Consider using #setUpRemoteFile if a download is requested.", +// urlToDownload); + } + } + + private static Uri getMobstoreUriForRemoteFile(String urlToDownload) { + return AndroidUri.builder(ApplicationProvider.getApplicationContext()) + .setModule("fakemddtest") + .setRelativePath(String.valueOf(Integer.valueOf(urlToDownload.hashCode()))) + .build(); + } + + @Override + public ListenableFuture<Boolean> removeFileGroup(RemoveFileGroupRequest removeFileGroupRequest) { + Throwable removeFileGroupThrowable = throwableMap.get(MethodType.REMOVE_FILE_GROUP); + if (removeFileGroupThrowable != null) { + return Futures.immediateFailedFuture(removeFileGroupThrowable); + } + removeFileGroupParamsList.add(removeFileGroupRequest); + return PropagatedFutures.submitAsync( + () -> Futures.immediateFuture(true), sequentialControlExecutor); + } + + @Override + public ListenableFuture<RemoveFileGroupsByFilterResponse> removeFileGroupsByFilter( + RemoveFileGroupsByFilterRequest removeFileGroupsByFilterRequest) { + return PropagatedFutures.submitAsync( + () -> + Futures.immediateFuture( + RemoveFileGroupsByFilterResponse.newBuilder().setRemovedFileGroupsCount(0).build()), + sequentialControlExecutor); + } + + @Override + public ListenableFuture<DataFileGroup> readDataFileGroup( + ReadDataFileGroupRequest readDataFileGroupRequest) { + return Futures.immediateFailedFuture(new UnsupportedOperationException()); + } + + @Override + public ListenableFuture<ClientFileGroup> getFileGroup(GetFileGroupRequest getFileGroupRequest) { + // Construct GroupKey from getFileGroupRequest. + GroupKey.Builder groupKeyBuilder = GroupKey.newBuilder(); + groupKeyBuilder.setGroupName(getFileGroupRequest.groupName()); + if (getFileGroupRequest.accountOptional().isPresent()) { + groupKeyBuilder.setAccount( + AccountUtil.serialize(getFileGroupRequest.accountOptional().get())); + } + if (getFileGroupRequest.variantIdOptional().isPresent()) { + groupKeyBuilder.setVariantId(getFileGroupRequest.variantIdOptional().get()); + } + GroupKey groupKey = groupKeyBuilder.build(); + + // Throw exception if a throwable is set. + Throwable getFileGroupThrowable = + methodTypeGroupKeyToThrowableTable.get(MethodType.GET_FILE_GROUP, groupKey); + if (getFileGroupThrowable == null) { + getFileGroupThrowable = throwableMap.get(MethodType.GET_FILE_GROUP); + } + if (getFileGroupThrowable != null) { + return Futures.immediateFailedFuture(getFileGroupThrowable); + } + getFileGroupParamsList.add(getFileGroupRequest); + return PropagatedFutures.submitAsync( + () -> { + List<ClientFileGroup> fileGroupList = + getMatchingFileGroups(groupKeyBuilder.build(), downloadedFileGroupList); + return Futures.immediateFuture(Iterables.getFirst(fileGroupList, null)); + }, + sequentialControlExecutor); + } + + @Override + public ListenableFuture<ImmutableList<ClientFileGroup>> getFileGroupsByFilter( + GetFileGroupsByFilterRequest getFileGroupsByFilterRequest) { + return PropagatedFutures.submitAsync( + () -> { + List<ClientFileGroup> allFileGroups = new ArrayList<>(downloadedFileGroupList); + allFileGroups.addAll(pendingFileGroupList); + + if (getFileGroupsByFilterRequest.includeAllGroups()) { + return Futures.immediateFuture(ImmutableList.copyOf(allFileGroups)); + } + + GroupKey.Builder groupKeyBuilder = GroupKey.newBuilder(); + if (getFileGroupsByFilterRequest.groupNameOptional().isPresent()) { + groupKeyBuilder.setGroupName(getFileGroupsByFilterRequest.groupNameOptional().get()); + } + if (getFileGroupsByFilterRequest.accountOptional().isPresent()) { + groupKeyBuilder.setAccount( + AccountUtil.serialize(getFileGroupsByFilterRequest.accountOptional().get())); + } + + return Futures.immediateFuture( + ImmutableList.copyOf(getMatchingFileGroups(groupKeyBuilder.build(), allFileGroups))); + }, + sequentialControlExecutor); + } + + @Override + public ListenableFuture<Void> importFiles(ImportFilesRequest importFilesRequest) { + return Futures.immediateVoidFuture(); + } + + /** + * If a file is set using setUpRemoteFile for {@code urlToDownload}, the contents will be copied + * to {@code destinationFileUri}. + */ + private void downloadFileIfSet(String urlToDownload, Uri destinationFileUri) throws IOException { + if (!remoteFilesMap.containsKey(urlToDownload)) { +// logger.atWarning().log( +// "No file set for %s using setUpRemoteFile. Download request is a no-op.", urlToDownload); + return; + } + + if (!storageOptional.isPresent()) { +// logger.atSevere().log("Storage not set."); + return; + } + + try (OutputStream out = + storageOptional.get().open(destinationFileUri, WriteStreamOpener.create())) { + out.write(remoteFilesMap.get(urlToDownload)); + } + } + + /** + * Copies file to the singleFileDownloadRequest#destinationFileUri if set using {@code + * setUpRemoteFile} + * + * <p>Storage needs to be present to copy the file to destinationFileUri and corresponding backend + * needs to be added to the storage. Throws UnsupportedFileStorageOperation if corresponding + * backend is not set. + */ + @Override + public ListenableFuture<Void> downloadFile(SingleFileDownloadRequest singleFileDownloadRequest) { + Throwable throwable = throwableMap.get(MethodType.DOWNLOAD_FILE); + if (throwable != null) { + return Futures.immediateFailedFuture(throwable); + } + return PropagatedFutures.submitAsync( + () -> { + try { + downloadFileIfSet( + singleFileDownloadRequest.urlToDownload(), + singleFileDownloadRequest.destinationFileUri()); + } catch (IOException e) { + return Futures.immediateFailedFuture(e); + } + + return Futures.immediateVoidFuture(); + }, + sequentialControlExecutor); + } + + @Override + public ListenableFuture<ClientFileGroup> downloadFileGroup( + DownloadFileGroupRequest downloadFileGroupRequest) { +// logger.atInfo().log("#downloadFileGroup: %s", downloadFileGroupRequest); + downloadFileGroupParamsList.add(downloadFileGroupRequest); + return PropagatedFutures.submitAsync( + () -> downloadFileGroupInternal(downloadFileGroupRequest), sequentialControlExecutor); + } + + @Override + public ListenableFuture<ClientFileGroup> downloadFileGroupWithForegroundService( + DownloadFileGroupRequest downloadFileGroupRequest) { +// logger.atInfo().log("#downloadFileGroupWithForegroundService: %s", downloadFileGroupRequest); + downloadFileGroupWithForegroundServiceParamsList.add(downloadFileGroupRequest); + return PropagatedFutures.submitAsync( + () -> downloadFileGroupInternal(downloadFileGroupRequest), sequentialControlExecutor); + } + + private ListenableFuture<ClientFileGroup> downloadFileGroupInternal( + DownloadFileGroupRequest downloadFileGroupRequest) { +// logger.atConfig().log("#downloadFileGroupInternal: %s", downloadFileGroupRequest); + GroupKey.Builder groupKeyBuilder = GroupKey.newBuilder(); + groupKeyBuilder.setGroupName(downloadFileGroupRequest.groupName()); + if (downloadFileGroupRequest.accountOptional().isPresent()) { + groupKeyBuilder.setAccount( + AccountUtil.serialize(downloadFileGroupRequest.accountOptional().get())); + } + if (downloadFileGroupRequest.variantIdOptional().isPresent()) { + groupKeyBuilder.setVariantId(downloadFileGroupRequest.variantIdOptional().get()); + } + + GroupKey groupKey = groupKeyBuilder.build(); + + List<ClientFileGroup> fileGroupList = getMatchingFileGroups(groupKey, downloadedFileGroupList); + if (!fileGroupList.isEmpty()) { + return Futures.immediateFuture(fileGroupList.get(0)); + } + + fileGroupList = getMatchingFileGroups(groupKey, pendingFileGroupList); + // If there is no match found in downloaded list, look for in pending list and update the + // status. + if (!fileGroupList.isEmpty()) { + ClientFileGroup fileGroup = fileGroupList.get(0); + ClientFileGroup downloadedFileGroup = + fileGroup.toBuilder().setStatus(ClientFileGroup.Status.DOWNLOADED).build(); + pendingFileGroupList.remove(fileGroup); + downloadedFileGroupList.add(downloadedFileGroup); + return Futures.immediateFuture(downloadedFileGroup); + } + + return Futures.immediateFailedFuture( + DownloadException.builder() + .setDownloadResultCode(DownloadResultCode.GROUP_NOT_FOUND_ERROR) + .build()); + } + + /** + * Copies file to the singleFileDownloadRequest#destinationFileUri if set using {@code + * setUpRemoteFile} + * + * <p>Storage needs to present to copy the file to destinationFileUri and corresponding backend + * needs to be added to the storage. Throws UnsupportedFileStorageOperation if corresponding + * backend is not set. + */ + @Override + public ListenableFuture<Void> downloadFileWithForegroundService( + SingleFileDownloadRequest singleFileDownloadRequest) { + Throwable throwable = throwableMap.get(MethodType.DOWNLOAD_FILE_FOREGROUND); + if (throwable != null) { + return Futures.immediateFailedFuture(throwable); + } + return PropagatedFutures.submitAsync( + () -> { + try { + downloadFileIfSet( + singleFileDownloadRequest.urlToDownload(), + singleFileDownloadRequest.destinationFileUri()); + } catch (IOException e) { + return Futures.immediateFailedFuture(e); + } + + return Futures.immediateVoidFuture(); + }, + sequentialControlExecutor); + } + + @Override + public void cancelForegroundDownload(String downloadKey) {} + + @Override + public ListenableFuture<Void> maintenance() { + return Futures.immediateVoidFuture(); + } + + @Override + public ListenableFuture<Void> collectGarbage() { + return Futures.immediateVoidFuture(); + } + + @Override + public void schedulePeriodicTasks() {} + + @Override + public ListenableFuture<Void> schedulePeriodicBackgroundTasks() { + return Futures.immediateVoidFuture(); + } + + @Override + public ListenableFuture<Void> schedulePeriodicBackgroundTasks( + Optional<Map<String, ConstraintOverrides>> constraintOverridesMap) { + return Futures.immediateVoidFuture(); + } + + @Override + public ListenableFuture<Void> cancelPeriodicBackgroundTasks() { + return Futures.immediateVoidFuture(); + } + + @Override + public ListenableFuture<Void> handleTask(String tag) { + handleTaskParamsList.add(tag); + return Futures.immediateVoidFuture(); + } + + @Override + public ListenableFuture<Void> clear() { + return Futures.immediateVoidFuture(); + } + + @Override + public String getDebugInfoAsString() { + return ""; + } + + @Override + public ListenableFuture<Void> reportUsage(UsageEvent usageEvent) { + return Futures.immediateVoidFuture(); + } +} diff --git a/javatests/com/google/android/libraries/mobiledatadownload/testing/FakeTimeSource.java b/javatests/com/google/android/libraries/mobiledatadownload/testing/FakeTimeSource.java index c8e7fa8..20ef4f3 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/testing/FakeTimeSource.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/testing/FakeTimeSource.java @@ -16,6 +16,7 @@ package com.google.android.libraries.mobiledatadownload.testing; import com.google.android.libraries.mobiledatadownload.TimeSource; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -23,23 +24,36 @@ import java.util.concurrent.atomic.AtomicLong; public final class FakeTimeSource implements TimeSource { private final AtomicLong currentMillis = new AtomicLong(); + private final AtomicLong elapsedNanos = new AtomicLong(); @Override public long currentTimeMillis() { return currentMillis.get(); } + @Override + public long elapsedRealtimeNanos() { + return elapsedNanos.get(); + } + /** Advances the current time and returns {@code this}. */ + @CanIgnoreReturnValue public FakeTimeSource advance(long interval, TimeUnit units) { long millis = units.toMillis(interval); if (millis < 0) { throw new IllegalArgumentException("Can't advance negative duration: " + millis); } currentMillis.getAndAdd(millis); + long nanos = units.toNanos(interval); + if (nanos < 0) { + throw new IllegalArgumentException("Can't advance negative duration: " + nanos); + } + elapsedNanos.getAndAdd(nanos); return this; } /** Sets the current time and returns {@code this}. */ + @CanIgnoreReturnValue public FakeTimeSource set(long millis) { if (millis < 0) { throw new IllegalArgumentException("Can't set before unix epoch:" + millis); @@ -47,4 +61,14 @@ public final class FakeTimeSource implements TimeSource { currentMillis.set(millis); return this; } + + /** Sets the elapsed time and returns {@code this}. */ + @CanIgnoreReturnValue + public FakeTimeSource setElapsedNanos(long nanos) { + if (nanos < 0) { + throw new IllegalArgumentException("Negative elapsed time: " + nanos); + } + elapsedNanos.set(nanos); + return this; + } } diff --git a/javatests/com/google/android/libraries/mobiledatadownload/testing/MddNotificationCapture.java b/javatests/com/google/android/libraries/mobiledatadownload/testing/MddNotificationCapture.java index 96454e7..5cb76b5 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/testing/MddNotificationCapture.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/testing/MddNotificationCapture.java @@ -102,7 +102,7 @@ public interface MddNotificationCapture { void assertFailedNotificationCaptured(String title); - void assertPausedNotificationCaptured(String title); + void assertPausedNotificationCaptured(String title, boolean wifiOnly); void assertNoNotificationsCaptured(); @@ -150,10 +150,12 @@ public interface MddNotificationCapture { } @Override - public void assertPausedNotificationCaptured(String title) { + public void assertPausedNotificationCaptured(String title, boolean wifiOnly) { assertNotificationCapturedMatches( title, - NotificationUtil.getDownloadPausedMessage(context), + wifiOnly + ? NotificationUtil.getDownloadPausedWifiMessage(context) + : NotificationUtil.getDownloadPausedMessage(context), android.R.drawable.stat_sys_download); } @@ -171,11 +173,9 @@ public interface MddNotificationCapture { assertThat(iconMatches) .comparingElementsUsing( Correspondence.<String, Integer>transforming( - match -> { - // Our regex should capture only valid hexadecimal values - int iconResId = Integer.parseInt(match, 16); - return iconResId; - }, + match -> + // Our regex should capture only valid hexadecimal values + Integer.parseInt(match, 16), "convert to resource id")) .containsNoneIn(MDD_ICON_IDS); } @@ -272,12 +272,14 @@ public interface MddNotificationCapture { } @Override - public void assertPausedNotificationCaptured(String title) { + public void assertPausedNotificationCaptured(String title, boolean wifiOnly) { assertThat(notifications) .comparingElementsUsing( createMatcherForNotification( title, - NotificationUtil.getDownloadPausedMessage(context), + wifiOnly + ? NotificationUtil.getDownloadPausedWifiMessage(context) + : NotificationUtil.getDownloadPausedMessage(context), android.R.drawable.stat_sys_download, "is a paused notification")) .contains(true); diff --git a/javatests/com/google/android/libraries/mobiledatadownload/testing/MddTestDependencies.java b/javatests/com/google/android/libraries/mobiledatadownload/testing/MddTestDependencies.java new file mode 100644 index 0000000..921927b --- /dev/null +++ b/javatests/com/google/android/libraries/mobiledatadownload/testing/MddTestDependencies.java @@ -0,0 +1,169 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.testing; + +import android.content.Context; + +import com.google.android.libraries.mobiledatadownload.TimeSource; +import com.google.android.libraries.mobiledatadownload.internal.logging.LoggingStateStore; +import com.google.android.libraries.mobiledatadownload.internal.logging.SharedPreferencesLoggingState; +import com.google.common.base.Optional; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +import java.util.Random; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; + +/** + * Utility class that provides support for building MDD with different types of dependencies for + * Testing. + * + * <p>If multiple type of dependencies need to be supported across tests, they can be defined here + * so all tests can rely on a single definition. This is useful for parameterizing tests, such as + * the case for ControlExecutor: + * + * <pre>{@code + * // In the test, define a parameter for ExecutorType + * @TestParameter ExecutorType controlExecutorType; + * + * // When building MDD in the test, rely on the shared provider: + * MobileDataDownloadBuilder.newBuilder() + * .setControlExecutor(controlExecutorType.executor()) + * // include other dependencies... + * .build(); + * + * }</pre> + */ +public final class MddTestDependencies { + + private static final String ALPHABET = "abcdefghijklmnopqrstuvwxyz"; + private static final int INSTANCE_ID_CHAR_LIMIT = 10; + private static final Random random = new Random(); + + private MddTestDependencies() { + } + + /** + * Generates a random instance id. + * + * <p>This prevents potential cross test conflicts from occurring since metadata will be siloed + * between tests. + */ + public static String randomInstanceId() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < INSTANCE_ID_CHAR_LIMIT; i++) { + sb.append(ALPHABET.charAt(random.nextInt(ALPHABET.length()))); + } + return sb.toString(); + } + + /** + * Type of executor passed when building MDD. + * + * <p>Used for parameterizing tests. + */ + public enum ExecutorType { + SINGLE_THREADED, + MULTI_THREADED; + + public ListeningExecutorService executor() { + switch (this) { + case SINGLE_THREADED: + return MoreExecutors.listeningDecorator( + Executors.newSingleThreadExecutor( + new ThreadFactoryBuilder().setNameFormat( + "MddSingleThreaded-%d").build())); + case MULTI_THREADED: + return MoreExecutors.listeningDecorator( + Executors.newCachedThreadPool( + new ThreadFactoryBuilder().setNameFormat( + "MddMultiThreaded-%d").build())); + } + throw new AssertionError("ExecutorType unsupported"); + } + } + + /** + * Differentiates between Downloader Configurations. + * + * <p>Used for parameterizing tests, as well as for making configuration-specific test + * assertions. + */ +// public enum DownloaderConfigurationType { +// V2_PLATFORM; +// +// public Supplier<FileDownloader> fileDownloaderSupplier( +// Context context, +// ListeningExecutorService controlExecutor, +// ListeningScheduledExecutorService downloadExecutor, +// SynchronousFileStorage fileStorage, +// Flags flags, +// Optional<DownloadProgressMonitor> downloadProgressMonitor, +// Optional<String> instanceId) { +// +// // Set up file downloader supplier based on the configuration given +// switch (this) { +// case V2_PLATFORM: +// return () -> { +// return BaseFileDownloaderModule.createOffroad2FileDownloader( +// context, +// downloadExecutor, +// controlExecutor, +// fileStorage, +// new SharedPreferencesDownloadMetadata( +// context.getSharedPreferences("downloadmetadata", 0), +// controlExecutor), +// /* downloadProgressMonitor= */ downloadProgressMonitor, +// /* urlEngineOptional= */ Optional.absent(), +// /* exceptionHandlerOptional= */ Optional.absent(), +// /* authTokenProviderOptional= */ Optional.absent(), +//// /* cookieJarSupplierOptional= */ Optional.absent(), +// /* trafficTag= */ Optional.absent(), +// flags); +// }; +// } +// throw new AssertionError("Invalid DownloaderConfigurationType"); +// } +// } + + /** + * Differentiates between LoggingStateStore implementations. + * + * <p>Used for parameterizing tests, as well as for making configuration-specific test + * assertions. + */ + public enum LoggingStateStoreImpl { + SHARED_PREFERENCES; + + public LoggingStateStore loggingStateStore( + Context context, + Optional<String> instanceIdOptional, + TimeSource timeSource, + Executor backgroundExecutor, + Random random) { + + // Set up file downloader supplier based on the configuration given + switch (this) { + case SHARED_PREFERENCES: + return SharedPreferencesLoggingState.createFromContext( + context, instanceIdOptional, timeSource, backgroundExecutor, random); + } + throw new AssertionError("Invalid LoggingStateStoreImpl"); + } + } +} diff --git a/javatests/com/google/android/libraries/mobiledatadownload/testing/RobolectricFileDownloader.java b/javatests/com/google/android/libraries/mobiledatadownload/testing/RobolectricFileDownloader.java index 504c151..52ab804 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/testing/RobolectricFileDownloader.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/testing/RobolectricFileDownloader.java @@ -15,6 +15,7 @@ */ package com.google.android.libraries.mobiledatadownload.testing; +import static com.google.common.util.concurrent.Futures.immediateFailedFuture; import static com.google.common.util.concurrent.Futures.immediateVoidFuture; import android.net.Uri; @@ -23,9 +24,11 @@ import com.google.android.libraries.mobiledatadownload.downloader.DownloadReques import com.google.android.libraries.mobiledatadownload.downloader.FileDownloader; import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; import com.google.android.libraries.mobiledatadownload.file.backends.FileUri; +import com.google.common.util.concurrent.ExecutionSequencer; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.runtime.RunfilesPaths; +import java.io.IOException; import java.nio.file.Path; /** @@ -42,22 +45,43 @@ import java.nio.file.Path; public final class RobolectricFileDownloader implements FileDownloader { private final String testDataRelativePath; + private final SynchronousFileStorage fileStorage; + private final ListeningExecutorService executor; private final FileDownloader delegateDownloader; + // Sequence downloads to prevent any potential overwrites + private final ExecutionSequencer executionSequencer = ExecutionSequencer.create(); + public RobolectricFileDownloader( String testDataRelativePath, SynchronousFileStorage fileStorage, ListeningExecutorService executor) { this.testDataRelativePath = testDataRelativePath; + this.fileStorage = fileStorage; + this.executor = executor; this.delegateDownloader = new LocalFileDownloader(fileStorage, executor); } @Override public ListenableFuture<Void> startDownloading(DownloadRequest downloadRequest) { + return executionSequencer.submitAsync( + () -> startDownloadingInternal(downloadRequest), executor); + } + + private ListenableFuture<Void> startDownloadingInternal(DownloadRequest downloadRequest) { Uri fileUri = downloadRequest.fileUri(); String urlToDownload = downloadRequest.urlToDownload(); DownloadConstraints downloadConstraints = downloadRequest.downloadConstraints(); + // If the file already exists, return immediately + try { + if (fileStorage.exists(fileUri)) { + return immediateVoidFuture(); + } + } catch (IOException e) { + return immediateFailedFuture(e); + } + // We need to translate the real urlToDownload to the one representing the local file in // testdata folder. Uri uriToDownload = Uri.parse(urlToDownload.trim()); diff --git a/javatests/com/google/android/libraries/mobiledatadownload/testing/TestFileDownloader.java b/javatests/com/google/android/libraries/mobiledatadownload/testing/TestFileDownloader.java index fb42c1a..a7f9db2 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/testing/TestFileDownloader.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/testing/TestFileDownloader.java @@ -68,11 +68,6 @@ public final class TestFileDownloader implements FileDownloader { LogUtil.e("%s: Invalid urlToDownload %s", TAG, urlToDownload); return immediateVoidFuture(); } - if (uriToDownload.getPath().endsWith("odws1_empty.jar")) { - // TODO(b/222519077): this is necessary to adapt the real file URL to local testdata - uriToDownload = - Uri.parse(uriToDownload.getPath().substring(0, uriToDownload.getPath().length() - 4)); - } String testDataUrl = FileUri.builder() diff --git a/javatests/com/google/android/libraries/mobiledatadownload/testing/TestFlags.java b/javatests/com/google/android/libraries/mobiledatadownload/testing/TestFlags.java index 85c9648..5efada8 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/testing/TestFlags.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/testing/TestFlags.java @@ -61,6 +61,7 @@ public final class TestFlags implements Flags { public Optional<Boolean> enableDownloadStageExperimentIdPropagation = Optional.absent(); public Optional<Boolean> enableIsolatedStructureVerification = Optional.absent(); public Optional<Boolean> enableRngBasedDeviceStableSampling = Optional.absent(); + public Optional<Boolean> enableFileDownloadDedupByFileKey = Optional.absent(); public Optional<Long> maintenanceGcmTaskPeriod = Optional.absent(); public Optional<Long> chargingGcmTaskPeriod = Optional.absent(); public Optional<Long> cellularChargingGcmTaskPeriod = Optional.absent(); @@ -291,6 +292,11 @@ public final class TestFlags implements Flags { } @Override + public boolean enableFileDownloadDedupByFileKey() { + return enableFileDownloadDedupByFileKey.or(delegate.enableRngBasedDeviceStableSampling()); + } + + @Override public long maintenanceGcmTaskPeriod() { return maintenanceGcmTaskPeriod.or(delegate.maintenanceGcmTaskPeriod()); } diff --git a/javatests/com/google/android/libraries/mobiledatadownload/testing/TestHttpServer.java b/javatests/com/google/android/libraries/mobiledatadownload/testing/TestHttpServer.java index 466d6d2..46f80d2 100644 --- a/javatests/com/google/android/libraries/mobiledatadownload/testing/TestHttpServer.java +++ b/javatests/com/google/android/libraries/mobiledatadownload/testing/TestHttpServer.java @@ -90,12 +90,12 @@ public final class TestHttpServer { /** Registers a handler that binds onto a text file for an endpoint pattern. */ public void registerTextFile(String pattern, String filepath) { - registerFile(pattern, filepath, TEXT_CONTENT_TYPE, /* eTagOptional = */ Optional.absent()); + registerFile(pattern, filepath, TEXT_CONTENT_TYPE, /* eTagOptional= */ Optional.absent()); } /** Registers a handler that binds onto a file for an endpoint pattern. */ public void registerBinaryFile(String pattern, String filepath) { - registerFile(pattern, filepath, BINARY_CONTENT_TYPE, /*eTagOptional=*/ Optional.absent()); + registerFile(pattern, filepath, BINARY_CONTENT_TYPE, /* eTagOptional= */ Optional.absent()); } /** @@ -131,7 +131,7 @@ public final class TestHttpServer { public Uri.Builder startServer() throws IOException { serverSocket = new ServerSocket( - /*port=*/ userDesignatedPort, /*backlog=*/ 0, InetAddress.getByName(TEST_HOST)); + /* port= */ userDesignatedPort, /* backlog= */ 0, InetAddress.getByName(TEST_HOST)); serverThread = new Thread( () -> { diff --git a/javatests/config/robolectric.properties b/javatests/config/robolectric.properties new file mode 100644 index 0000000..83d7549 --- /dev/null +++ b/javatests/config/robolectric.properties @@ -0,0 +1,15 @@ +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +sdk=NEWEST_SDK
\ No newline at end of file diff --git a/mobile-data-download.iml b/mobile-data-download.iml deleted file mode 100644 index afa4d1e..0000000 --- a/mobile-data-download.iml +++ /dev/null @@ -1,13 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<module type="JAVA_MODULE" version="4"> - <component name="NewModuleRootManager" inherit-compiler-output="true"> - <exclude-output /> - <content url="file://$MODULE_DIR$"> - <sourceFolder url="file://$MODULE_DIR$/android-annotation-stubs" isTestSource="false" packagePrefix="__PACKAGE__" /> - <sourceFolder url="file://$MODULE_DIR$/java" isTestSource="false" /> - <sourceFolder url="file://$MODULE_DIR$/javatests" isTestSource="false" /> - </content> - <orderEntry type="inheritedJdk" /> - <orderEntry type="sourceFolder" forTests="false" /> - </component> -</module>
\ No newline at end of file diff --git a/proto/Android.bp b/proto/Android.bp index f3f5e71..40a3017 100644 --- a/proto/Android.bp +++ b/proto/Android.bp @@ -41,5 +41,7 @@ java_library { apex_available: [ "//apex_available:platform", "com.android.adservices", + "com.android.extservices", + "com.android.ondevicepersonalization", ], } diff --git a/proto/BUILD b/proto/BUILD index 52b5e18..09df231 100644 --- a/proto/BUILD +++ b/proto/BUILD @@ -1,4 +1,7 @@ +load("//third_party/bazel_rules/rules_java/java:defs.bzl", "java_proto_library") + package( + default_applicable_licenses = ["//:license"], default_visibility = ["//visibility:public"], licenses = ["notice"], ) @@ -28,6 +31,11 @@ proto_library( alwayslink = 1, ) +kt_jvm_lite_proto_library( + name = "download_config_kt_proto_lite", + deps = [":download_config_proto"], +) + java_lite_proto_library( name = "download_config_java_proto_lite", deps = [":download_config_proto"], @@ -39,7 +47,48 @@ proto_library( cc_api_version = 2, ) +java_proto_library( + name = "transform_java_proto", + deps = [":transform_proto"], +) + java_lite_proto_library( name = "transform_java_proto_lite", deps = [":transform_proto"], ) + +proto_library( + name = "logs_proto", + srcs = ["logs.proto"], + cc_api_version = 2, + deps = [ + ":log_enums_proto", + ], +) + +java_lite_proto_library( + name = "logs_java_proto_lite", + deps = [":logs_proto"], +) + +proto_library( + name = "log_enums_proto", + srcs = ["log_enums.proto"], + cc_api_version = 2, +) + +java_lite_proto_library( + name = "log_enums_java_proto_lite", + deps = [":log_enums_proto"], +) + +proto_library( + name = "atoms_proto", + srcs = ["atoms.proto"], + cc_api_version = 2, +) + +java_lite_proto_library( + name = "atoms_java_proto_lite", + deps = [":atoms_proto"], +) diff --git a/proto/atoms.proto b/proto/atoms.proto new file mode 100644 index 0000000..69e7c41 --- /dev/null +++ b/proto/atoms.proto @@ -0,0 +1,70 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +syntax = "proto2"; + +package mobiledatadownload.logs; + +option java_multiple_files = true; +option java_package = "com.google.mobiledatadownload"; +option java_outer_classname = "AtomsProto"; + +/** + * These protos are duplicates of the MobileDataDownload protos logged as + * MODE_BYTES in <internal>. + * TODO(b/243579271): remove this duplication + */ + +/** Shared data among MobileDataDownload statistics. Not meant to be a top level + * atom proto.*/ +message MobileDataDownloadFileGroupStats { + // The name of the file group. This a string set server side used to retrieve + // the files. Does not contain PII. + optional string file_group_name = 1; + // Allows the clients to identify a file group based on a given set of + // properties. This string is set server side and does not contain PII. + optional string variant_id = 2; + // Identifier for the data file group created to identify the version of the + // file group. + optional int64 build_id = 3; + // The number of files in the file group. + optional int32 file_count = 4; + // Whether the file group has an account associated with it. + optional bool has_account = 5; + // Inverse of the sampling rate used to sample this event. + optional int32 sampling_interval = 6; + // Note: we do not have owner_package since that's already transmitted. +} + +/** + * Container for MobileDataDownloadFileGroupStorageStats + */ +message MobileDataDownloadStorageStats { + repeated MobileDataDownloadFileGroupStorageStats + mobile_data_download_file_group_storage_stats = 1; +} + +/** + * Storage stats for a single file group. This is logged as a nested field and + * is not meant to be logged as a top level proto. + */ +message MobileDataDownloadFileGroupStorageStats { + // The total number of bytes used by this file group + optional int64 total_bytes_used = 1; + // The total number of inline file bytes used by this file group + optional int64 total_inline_bytes_used = 2; + // The number of bytes used for the downloaded version of the file group + optional int64 downloaded_group_bytes_used = 3; + // Specifies which file group this storage is associated with + optional MobileDataDownloadFileGroupStats file_group_stats = 4; +} diff --git a/proto/client_config.proto b/proto/client_config.proto index 7ae1e98..f6260c7 100644 --- a/proto/client_config.proto +++ b/proto/client_config.proto @@ -17,7 +17,7 @@ package com.google.android.libraries.mdi.download; import "google/protobuf/any.proto"; -//option jspb_use_correct_proto2_semantics = false; // <internal> TODO +//option jspb_use_correct_proto2_semantics = false; // <internal> option java_package = "com.google.mobiledatadownload"; option java_outer_classname = "ClientConfigProto"; option objc_class_prefix = "ICN"; diff --git a/proto/download_config.proto b/proto/download_config.proto index c4c7e34..1b88c23 100644 --- a/proto/download_config.proto +++ b/proto/download_config.proto @@ -21,7 +21,7 @@ import "transform.proto"; option java_package = "com.google.mobiledatadownload"; option java_outer_classname = "DownloadConfigProto"; option objc_class_prefix = "Icing"; -//option go_api_flag = "OPEN_TO_OPAQUE_HYBRID"; // See <internal>. TODO +//option go_api_flag = "OPEN_TO_OPAQUE_HYBRID"; // See <internal>. // The top-level proto for Mobile Data Download (<internal>). message DownloadConfig { @@ -132,9 +132,21 @@ message DataFileGroup { // Ex: 172800 // 2 Days optional int64 stale_lifetime_secs = 3; - // The timestamp at which this filegroup should be deleted, even if it is - // still active, specified in seconds since epoch. - // NOTE: MDD will delete the file group version within a day of this time. + // The timestamp at which this filegroup should be deleted specified in + // seconds since epoch. This is a hard deadline and can be applied to file + // groups still in the ACTIVE state. If the value is 0, that is the same as + // unset (no expiration). Expiration is performed at next cleanup time, which + // is typically daily. Therefore, file groups may remain even after expired, + // and may do so indefinitely if cleanup is not scheduled. + // + // NOTE this is not the way to delete a file group. For example, setting an + // expiration date in the past will fail, potentially leaving an unexpired + // file group in place indefinitely. Use the MDD removeFileGroup API for that + // on device. From the server, the way to delete a file group is to add a new + // one with the same name, but with no files (this functions as a tombstone). + // + // NOTE b/252890898 for behavior on CastOS (cMDD) + // NOTE b/252885626 for missing support for delete in MobServe Ingress optional int64 expiration_date = 11; // Specify the conditions under which the file group should be downloaded. @@ -227,8 +239,8 @@ message DataFile { DEFAULT = 0; // No checksum is provided. - // This is NOT currently supported by iMDD. Please contact <internal>@ if - // you need this feature. + // This is NOT currently supported by iMDD. Please contact <internal>@ if you + // need this feature. NONE = 1; // This is currently only supported by cMDD. If you need it for Android or @@ -518,6 +530,8 @@ message ManifestConfig { // prefix encoding, however, for the S2CellIds the high-order bits // encode the face-ID and as a result we often end up with large // numbers. +// optional fixed64 s2_cell_id = 1 [ +// (datapol.semantic_type) = ST_LOCATION optional fixed64 s2_cell_id = 1; } diff --git a/proto/log_enums.proto b/proto/log_enums.proto new file mode 100644 index 0000000..a86c611 --- /dev/null +++ b/proto/log_enums.proto @@ -0,0 +1,173 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +syntax = "proto3"; + +package mobiledatadownload.logs; + +option java_package = "com.google.mobiledatadownload"; +option java_outer_classname = "LogEnumsProto"; + +// MDD client side events used for logging with MddLogData. +// +// Each feature gets a range of 1000 enums starting at X000. 1st enum specifies +// if the feature is enabled. Subsequent 999 enums can be used to define events +// within the feature. Unused enums in the range are left for future use for +// the *same* feature. +// If a feature ever exhausts it's quota of enums, it should be migrated to a +// new range of contiguous 2000 enums by deprecating the existing enums. +// +// Enums should never be deleted or reused, but they can be renamed*. Old enums +// should be left in their position with [deprecated=true] attribute. +// +// * For renaming enums, see <internal> +message MddClientEvent { + enum Code { + // Do not use this default value. + EVENT_CODE_UNSPECIFIED = 0; + + // Events for Mobile Data Download (<internal>) (1000-1999). + // Next enum for data download: 1114 + + // Log in a periodic tasks. + // Logged with DataDownloadFileGroupStats, MddFileGroupStatus. + DATA_DOWNLOAD_FILE_GROUP_STATUS = 1044; + + // Log MddStorageStats in daily maintenance. + DATA_DOWNLOAD_STORAGE_STATS = 1055; + + // MDD download result log. + DATA_DOWNLOAD_RESULT_LOG = 1068; + + reserved 1000 to 1043, 1045 to 1054, 1056 to 1067, 1069 to 1113; + + reserved 2000 to 2999, 3000 to 3999, 4000 to 4099, 4100 to 4199, + 5000 to 5999, 6000 to 6999, 7000 to 7999, 8000 to 8999, 9000 to 9999, + 10000 to 10999, 11000 to 11999, 12000 to 12999, 13000, 13999, + 14000 to 14999, 15000 to 15999, 16000 to 16999, 17000 to 17999, + 18000 to 18999, 19000 to 19999; + } +} + +message MddFileGroupDownloadStatus { + enum Code { + INVALID = 0; + COMPLETE = 1; + PENDING = 2; + FAILED = 3; + } +} + +// Result of MDD download api call. +message MddDownloadResult { + enum Code { + UNSPECIFIED = 0; // unset value + + // File downloaded successfully. + SUCCESS = 1; + + // The error we don't know. + UNKNOWN_ERROR = 2; + + // The errors from the android downloader v1 outside MDD, which comes from: + // <internal> + // The block 100-199 (included) is reserved for android downloader v1. + // Next tag: 112 + ANDROID_DOWNLOADER_UNKNOWN = 100; + ANDROID_DOWNLOADER_CANCELED = 101; + ANDROID_DOWNLOADER_INVALID_REQUEST = 102; + ANDROID_DOWNLOADER_HTTP_ERROR = 103; + ANDROID_DOWNLOADER_REQUEST_ERROR = 104; + ANDROID_DOWNLOADER_RESPONSE_OPEN_ERROR = 105; + ANDROID_DOWNLOADER_RESPONSE_CLOSE_ERROR = 106; + ANDROID_DOWNLOADER_NETWORK_IO_ERROR = 107; + ANDROID_DOWNLOADER_DISK_IO_ERROR = 108; + ANDROID_DOWNLOADER_FILE_SYSTEM_ERROR = 109; + ANDROID_DOWNLOADER_UNKNOWN_IO_ERROR = 110; + ANDROID_DOWNLOADER_OAUTH_ERROR = 111; + + // The errors from the android downloader v2 outside MDD, which comes from: + // <internal> + // The block 200-299 (included) is reserved for android downloader v2. + // Next tag: 201 + ANDROID_DOWNLOADER2_ERROR = 200; + + // The data file group has not been added to MDD by the time the caller + // makes download API call. + GROUP_NOT_FOUND_ERROR = 300; + + // The DownloadListener is present but the DownloadMonitor is not provided. + DOWNLOAD_MONITOR_NOT_PROVIDED_ERROR = 301; + + // Errors from unsatisfied download preconditions. + INSECURE_URL_ERROR = 302; + LOW_DISK_ERROR = 303; + + // Errors from download preparation. + UNABLE_TO_CREATE_FILE_URI_ERROR = 304; + SHARED_FILE_NOT_FOUND_ERROR = 305; + MALFORMED_FILE_URI_ERROR = 306; + UNABLE_TO_CREATE_MOBSTORE_RESPONSE_WRITER_ERROR = 307; + + // Errors from file validation. + UNABLE_TO_VALIDATE_DOWNLOAD_FILE_ERROR = 308; + DOWNLOADED_FILE_NOT_FOUND_ERROR = 309; + DOWNLOADED_FILE_CHECKSUM_MISMATCH_ERROR = 310; + CUSTOM_FILEGROUP_VALIDATION_FAILED = 330; + + // Errors from download transforms. + UNABLE_TO_SERIALIZE_DOWNLOAD_TRANSFORM_ERROR = 311; + DOWNLOAD_TRANSFORM_IO_ERROR = 312; + FINAL_FILE_CHECKSUM_MISMATCH_ERROR = 313; + + // Errors from delta download. + DELTA_DOWNLOAD_BASE_FILE_NOT_FOUND_ERROR = 314; + DELTA_DOWNLOAD_DECODE_IO_ERROR = 315; + + // The error occurs after the file is ready. + UNABLE_TO_UPDATE_FILE_STATE_ERROR = 316; + + // Fail to update the file group metadata. + UNABLE_TO_UPDATE_GROUP_METADATA_ERROR = 317; + + // Errors from sharing files with the blob storage. + // Failed to update the metadata max_expiration_date. + UNABLE_TO_UPDATE_FILE_MAX_EXPIRATION_DATE = 318; + UNABLE_SHARE_FILE_BEFORE_DOWNLOAD_ERROR = 319; + UNABLE_SHARE_FILE_AFTER_DOWNLOAD_ERROR = 320; + + // Download errors related to isolated file structure + UNABLE_TO_REMOVE_SYMLINK_STRUCTURE = 321; + UNABLE_TO_CREATE_SYMLINK_STRUCTURE = 322; + + // Download errors related to importing inline files + UNABLE_TO_RESERVE_FILE_ENTRY = 323; + INVALID_INLINE_FILE_URL_SCHEME = 324; + INLINE_FILE_IO_ERROR = 327; + MISSING_INLINE_DOWNLOAD_PARAMS = 328; + MISSING_INLINE_FILE_SOURCE = 329; + + // Download errors related to URL parsing + MALFORMED_DOWNLOAD_URL = 325; + UNSUPPORTED_DOWNLOAD_URL_SCHEME = 326; + + // Download errors for manifest file group populator. + MANIFEST_FILE_GROUP_POPULATOR_INVALID_FLAG_ERROR = 400; + MANIFEST_FILE_GROUP_POPULATOR_CONTENT_CHANGED_DURING_DOWNLOAD_ERROR = 401; + MANIFEST_FILE_GROUP_POPULATOR_PARSE_MANIFEST_FILE_ERROR = 402; + MANIFEST_FILE_GROUP_POPULATOR_DELETE_MANIFEST_FILE_ERROR = 403; + MANIFEST_FILE_GROUP_POPULATOR_METADATA_IO_ERROR = 404; + + reserved 1000 to 3000; + } +} diff --git a/proto/logs.proto b/proto/logs.proto new file mode 100644 index 0000000..b35aa07 --- /dev/null +++ b/proto/logs.proto @@ -0,0 +1,271 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// Logging protos for MobileDataDownload + +syntax = "proto2"; + +package mobiledatadownload.logs; + +import "log_enums.proto"; + +//option jspb_use_correct_proto2_semantics = false; // <internal> +option java_package = "com.google.mobiledatadownload"; +option java_outer_classname = "LogProto"; + +// Info about the Android client that logged. +// Next tag: 3 +message AndroidClientInfo { + // Version of the module we are currently running. aMDD will log its own + // version that it shares between GMSCore module and library. + + optional int32 module_version = 1 [default = -1]; + + // Package name of the hosting application. + // It is to differentiate logs from GMS service and library. + optional string host_package_name = 2; +} + +// Attributes of the device and/or MDD +// Recommended to log this message with each MDD log defined below. This will +// allow slicing MDD stats on the state of the device. +// +// TODO: Make Fields of this proto available as RASTA conditions for +// experimentation. +// +// Next tag: 3 +message MddDeviceInfo { + // Indicates low storage space condition on the device. + // Currently in O-, it is the result of Android's ACTION_DEVICE_STORAGE_LOW + // intent when the storage state was logged. + // For O+, MDD will define its own threshold for low storage: b/77856395 + optional bool device_storage_low = 1; + + reserved 2; +} + +// Metadata associated with each data download event specific to a file group. +// +// Next tag: 9 +message DataDownloadFileGroupStats { + // Name of the file group. + optional string file_group_name = 1; + + // Client set version number used to identify the file group. + // Note that this does not uniquely identify the contents of the file group. + // It simply reflects a snapshot of client config changes. + // For example: say there's a file group 'language-detector-model' that + // downloads a different file per user locale. + // data_file_group { + // file_group_name = 'language-detector-model' + // file_group_version_number = 1 + // file { + // url = 'en-model' + // } + // } + // data_file_group { + // file_group_name = 'language-detector-model' + // file_group_version_number = 1 + // file { + // url = 'es-model' + // } + // } + // Note that even though the actual contents of the file group are different + // for each locale, the version is the same because this config was pushed + // at the same snapshot. + optional int32 file_group_version_number = 2; + + // The package name of the group owner. + optional string owner_package = 3; + + // The total number of files in the file group. + // + // NOTE: This count is only included for storage and file group stats logging + optional int32 file_count = 4; + + // The number of inline files in the file group. + // + // NOTE: This count is only included for storage and file group stats logging + optional int32 inline_file_count = 8; + + // Whether the file group has an account associated with it or not. This will + // allow us to slice metrics by having account or not. For more info see + // <internal> + optional bool has_account = 5; + + // The build id for the file group. Unique identifier for a file group config + // created when using MDD Ingress API. + // For more details see <internal>. + optional int64 build_id = 6; + + // The VariantID of the DataFileGroup. This is set up server side via code + // review. For more details see <internal>. + // Examples: "en", "de-universal", etc. + optional string variant_id = 7; +} + +// The status of a MDD file group. This data is logged periodically to get +// a snapshot of the status of a file group on devices. +// Next tag: 5 +message MddFileGroupStatus { + // Download status of the whole file group. This is an AND over the + // download status of each file within the file group. + optional MddFileGroupDownloadStatus.Code file_group_download_status = 1; + + // Time since epoch when this file group was first added to mdd. + // + // Set to -1 if this time is unknown (for legacy groups). + // + // This matches the field "group_new_files_received_timestamp" in metadata + // <internal> + optional int64 group_added_timestamp_in_seconds = 2; + + // Time since epoch when this file group was downloaded by mdd. + // + // Set to -1 if this time is unknown (for legacy groups) and non-downloaded + // groups + // + // This matches the field "group_downloaded_timestamp_in_millis" in metadata + // <internal> + optional int64 group_downloaded_timestamp_in_seconds = 3; + + // Number of days since this status was last logged (number of days since + // daily maintenance was last run). + // + // Set to -1 if there is an invalid or unknown value. + // + // See <internal> for more info. + optional int32 days_since_last_log = 4; +} + +// Various health reports. +// Ideally, this should be defined as an empty message that allows extensions +// and each inner proto should be defined as its extension. +// TODO: Figure out if there are limitations in nano-proto that might +// not allow this. +// +// Next tag: 74 +message MddLogData { + // MDD data download file group stats. + optional DataDownloadFileGroupStats data_download_file_group_stats = 10; + + // Sampling interval used while logging this message. The default value 0 is + // not a valid value for messages using this filed since it a special value + // denoting that message should not be logged. Hence value of 0 means it was + // not filled in. + optional int64 sampling_interval = 21; + + // Additional info necessary for stable sampling. + optional StableSamplingInfo stable_sampling_info = 72; + + // Data download file group download status (logged periodically). + optional MddFileGroupStatus mdd_file_group_status = 32; + + // Attributes of the device and/or MDD at the time we log other stats. + optional MddDeviceInfo device_info = 40; + + // Android client info. + optional AndroidClientInfo android_client_info = 51; + + optional MddStorageStats mdd_storage_stats = 46; + + // MDD download result log. + optional MddDownloadResultLog mdd_download_result_log = 63; + + reserved 1 to 9, 11 to 20, 22 to 31, 33 to 39, 41 to 45, 47 to 50, 52 to 62, + 64 to 71, 73; +} + +// Info on sampling method used for log events. Stable sampling means if a +// device is selected to log, it will log all events. See <internal> +// Next tag: 5 +message StableSamplingInfo { + // Whether a stable sampling method (as described in <internal>) + // was used. + optional bool stable_sampling_used = 1; + + // When stable sampling was first enabled on the device. This will be useful + // when rolling out and processing logs over multiple days. + optional int64 stable_sampling_first_enabled_timestamp_ms = 2; + + // Whether or not this device would log with the 1% cohort. Devices in the 1% + // cohort are *always* logging, and will always log without further code + // changes. When a device has this set to true, it's expected that the device + // is *always* logging and the sampling rate should not be changed to + // something that results in this device being excluded from the logging group + // (see invalid_sampling_rate_used). + // + // Most dashboards/metrics depending on linking together multiple events from + // the same device should filter to devices/events that have this set to true + // (with the caveat that we won't use all data from all devices reporting). + // This is useful when we need to change sampling rates, e.g. for an + // experiment. + optional bool part_of_always_logging_group = 3; + + // If we are using stable sampling, we expect a sampling rate where '100 % + // sample_interval == 0'. This ensures that devices logging at 1 percent + // sampling interval, will continue to log at other chosen sampling rates too. + // This should only be false if we've incorrectly configured our sampling + // rates (e.g. a sampling rate of 101 would mean that the 1 percent cohort + // devices would not log). + optional bool invalid_sampling_rate_used = 4; +} + +// MDD download result log. +message MddDownloadResultLog { + optional MddDownloadResult.Code result = 1; + // File group information. + optional DataDownloadFileGroupStats data_download_file_group_stats = 2; +} + +// MDD Storage stats +// Next tag 9 +message MddStorageStats { + repeated DataDownloadFileGroupStats data_download_file_group_stats = 1; + + // NOTE: The four repeated fields total_bytes_used, total_inline_bytes_used, + // downloaded_group_bytes_used, and downloaded_group_inline_bytes_used have + // the same length and an element from all fields with the same index + // refers to the same file group. + + // total_bytes_used[x] represents the total bytes used on disk by the + // file group index x. + repeated uint64 total_bytes_used = 2; + + // total_inline_bytes_used[x] represents the total bytes used on disk by + // _inline_ files of file group index x. + repeated uint64 total_inline_bytes_used = 7 [packed = true]; + + // Similarly, the downloaded_group_bytes_used[x] + // represents the bytes used in the corresponding downloaded file group. + repeated uint64 downloaded_group_bytes_used = 3; + + // Similarly, the downloaded_group_inline_bytes_used[x] represents the + // bytes of _inline_ files used in the corresponding downloaded file group. + repeated uint64 downloaded_group_inline_bytes_used = 8 [packed = true]; + + // Total bytes used by all MDD file groups. + // Measured by adding up file sizes for all files that are known to MDD. + optional uint64 total_mdd_bytes_used = 4; + + // Total bytes used by MDD download directory. + optional uint64 total_mdd_directory_bytes_used = 5; + + // Number of days since this status was last logged (number of days since + // daily maintenance was last run). + // + // Set to -1 if there is an invalid or unknown value. + // + // See <internal> for more info. + optional int32 days_since_last_log = 6; +}
\ No newline at end of file diff --git a/proto/metadata.proto b/proto/metadata.proto index 4b815d2..6e6d8dc 100644 --- a/proto/metadata.proto +++ b/proto/metadata.proto @@ -41,7 +41,7 @@ message ExtraHttpHeader { // The tag number of extra fields should start from 1000 to reserve room for // growing DataFileGroup. // -// Next id: 1000 +// Next id: 1001 message DataFileGroupInternal { // Extra information that is kept on disk. // @@ -199,6 +199,15 @@ message DataFileGroupInternal { reserved 28; + // If a group enables preserve_filenames_and_isolate_files + // this property will contain the directory root of the isolated + // structure. Specifically, the property will be a string created from the + // group name and a hash of other identifying properties (account, variantid, + // buildid). + // + // currently only used in aMDD. + optional string isolated_directory_root = 1000; + reserved 4, 5, 7, 8, 9, 15, 18, 22, 24; } @@ -507,8 +516,23 @@ message GroupKey { // Whether or not all files in a fileGroup have been downloaded. optional bool downloaded = 4; - // The variant id of the group. A null or empty value indicates that the group - // does not have an associated variant. + // The variant id of the group for identification purposes. + // + // This is used to ensure that groups with different variants can have + // different entries in MDD metadata, and therefore have different lifecycles. + // + // Note that clients can choose to opt-in to a SINGLE_VARIANT flow where + // different variants replace each other on-device (only single variant can + // exist on a device at a time). In this case, an empty variant_id is set here + // so groups with different variants share the same GroupKey and are subject + // to the same lifecycle, even though the DataFileGroup does have a non-empty + // variant_id. + // + // Because of the SINGLE_VARIANT flow and because groups may still be added + // with no variant_id associated, using this property to tell if the + // associated file group has a variant_id is unreliable. Instead, the + // variant_id set within a DataFileGroup should be used as the source of truth + // about the group (such as when logging). optional string variant_id = 6; reserved 3; @@ -617,7 +641,7 @@ message NewFileKey { optional string checksum = 3; optional DataFileGroupInternal.AllowedReaders allowed_readers = 4; optional mobstore.proto.Transforms download_transforms = 5 - [deprecated = true]; + [deprecated = true]; } // This proto is used to store state for logging. See details at @@ -651,11 +675,26 @@ message LoggingState { // This proto is used to store state for logging that is specific to a File // Group. This includes network usage logging and maybe download tiers (for // <internal>). +// +// NEXT TAG: 7 message FileGroupLoggingState { + // GroupKey associated with a file group -- this is used to populate the group + // name and host package name. optional GroupKey group_key = 1; + + // The build_id associated with the file group. optional int64 build_id = 2; + + // The variant_id associated with the file group. + optional string variant_id = 6; + + // The file group version number associated with the file group. optional int32 file_group_version_number = 3; + + // The number of bytes downloaded over a cellular (metered) network. optional int64 cellular_usage = 4; + + // The number of bytes downloaded over a wifi (unmetered) network. optional int64 wifi_usage = 5; } |