diff options
author | Presubmit Automerger Backend <android-build-presubmit-automerger-backend@system.gserviceaccount.com> | 2022-08-24 21:52:47 +0000 |
---|---|---|
committer | Presubmit Automerger Backend <android-build-presubmit-automerger-backend@system.gserviceaccount.com> | 2022-08-24 21:52:47 +0000 |
commit | 14cfc440750104896054c8b04ad824c8b02fdb2e (patch) | |
tree | 6c8d1ce27b95491b46a91804fd62196f2f0dd0ee | |
parent | 28f22d5379d69862bbfe73f1b6c1be830ae95836 (diff) | |
parent | 980a37b132d26691e1f2e0851bf4c4aca3fea03c (diff) | |
download | mobile-data-download-14cfc440750104896054c8b04ad824c8b02fdb2e.tar.gz |
[automerge] Import MDD logging 2p: 980a37b132
Original change: https://googleplex-android-review.googlesource.com/c/platform/external/mobile-data-download/+/19682334
Bug: 219636547
Change-Id: Ic00af798a8e27c2eb870c74a294249238968ba8f
17 files changed, 1952 insertions, 361 deletions
diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/SharedPreferencesFileGroupsMetadata.java b/java/com/google/android/libraries/mobiledatadownload/internal/SharedPreferencesFileGroupsMetadata.java index bc407fd..9b1ba9a 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/SharedPreferencesFileGroupsMetadata.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/SharedPreferencesFileGroupsMetadata.java @@ -36,6 +36,7 @@ import com.google.errorprone.annotations.CheckReturnValue; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; import com.google.mobiledatadownload.internal.MetadataProto.GroupKeyProperties; + import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; @@ -54,7 +55,7 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta private static final String TAG = "SharedPreferencesFileGroupsMetadata"; private static final String MDD_FILE_GROUPS = FileGroupsMetadataUtil.MDD_FILE_GROUPS; private static final String MDD_FILE_GROUP_KEY_PROPERTIES = - FileGroupsMetadataUtil.MDD_FILE_GROUP_KEY_PROPERTIES; + FileGroupsMetadataUtil.MDD_FILE_GROUP_KEY_PROPERTIES; // TODO(b/144033163): Migrate the Garbage Collector File to PDS. @VisibleForTesting static final String MDD_GARBAGE_COLLECTION_FILE = "gms_icing_mdd_garbage_file"; @@ -67,11 +68,11 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta @Inject SharedPreferencesFileGroupsMetadata( - @ApplicationContext Context context, - TimeSource timeSource, - SilentFeedback silentFeedback, - @InstanceId Optional<String> instanceId, - @SequentialControlExecutor Executor sequentialControlExecutor) { + @ApplicationContext Context context, + TimeSource timeSource, + SilentFeedback silentFeedback, + @InstanceId Optional<String> instanceId, + @SequentialControlExecutor Executor sequentialControlExecutor) { this.context = context; this.timeSource = timeSource; this.silentFeedback = silentFeedback; @@ -86,66 +87,66 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta @Override public ListenableFuture<@NullableType DataFileGroupInternal> read(GroupKey groupKey) { - String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey, context); + String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey); SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); + SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); DataFileGroupInternal fileGroup = - SharedPreferencesUtil.readProto(prefs, serializedGroupKey, DataFileGroupInternal.parser()); + SharedPreferencesUtil.readProto(prefs, serializedGroupKey, DataFileGroupInternal.parser()); return Futures.immediateFuture(fileGroup); } @Override public ListenableFuture<Boolean> write(GroupKey groupKey, DataFileGroupInternal fileGroup) { - String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey, context); + String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey); SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); + SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); return Futures.immediateFuture( - SharedPreferencesUtil.writeProto(prefs, serializedGroupKey, fileGroup)); + SharedPreferencesUtil.writeProto(prefs, serializedGroupKey, fileGroup)); } @Override public ListenableFuture<Boolean> remove(GroupKey groupKey) { - String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey, context); + String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey); SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); + SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); return Futures.immediateFuture(SharedPreferencesUtil.removeProto(prefs, serializedGroupKey)); } @Override public ListenableFuture<@NullableType GroupKeyProperties> readGroupKeyProperties( - GroupKey groupKey) { - String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey, context); + GroupKey groupKey) { + String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey); SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences( - context, MDD_FILE_GROUP_KEY_PROPERTIES, instanceId); + SharedPreferencesUtil.getSharedPreferences( + context, MDD_FILE_GROUP_KEY_PROPERTIES, instanceId); GroupKeyProperties groupKeyProperties = - SharedPreferencesUtil.readProto(prefs, serializedGroupKey, GroupKeyProperties.parser()); + SharedPreferencesUtil.readProto(prefs, serializedGroupKey, GroupKeyProperties.parser()); return Futures.immediateFuture(groupKeyProperties); } @Override public ListenableFuture<Boolean> writeGroupKeyProperties( - GroupKey groupKey, GroupKeyProperties groupKeyProperties) { - String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey, context); + GroupKey groupKey, GroupKeyProperties groupKeyProperties) { + String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey); SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences( - context, MDD_FILE_GROUP_KEY_PROPERTIES, instanceId); + SharedPreferencesUtil.getSharedPreferences( + context, MDD_FILE_GROUP_KEY_PROPERTIES, instanceId); return Futures.immediateFuture( - SharedPreferencesUtil.writeProto(prefs, serializedGroupKey, groupKeyProperties)); + SharedPreferencesUtil.writeProto(prefs, serializedGroupKey, groupKeyProperties)); } @Override public ListenableFuture<List<GroupKey>> getAllGroupKeys() { List<GroupKey> groupKeyList = new ArrayList<>(); SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); + SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); SharedPreferences.Editor editor = null; for (String serializedGroupKey : prefs.getAll().keySet()) { try { @@ -175,36 +176,36 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta @Override public ListenableFuture<List<Pair<GroupKey, DataFileGroupInternal>>> getAllFreshGroups() { return Futures.transformAsync( - getAllGroupKeys(), - groupKeyList -> { - List<ListenableFuture<@NullableType DataFileGroupInternal>> groupReadFutures = - new ArrayList<>(); - for (GroupKey key : groupKeyList) { - groupReadFutures.add(read(key)); - } - return Futures.whenAllComplete(groupReadFutures) - .callAsync( - () -> { - List<Pair<GroupKey, DataFileGroupInternal>> retrievedGroups = new ArrayList<>(); - for (int i = 0; i < groupKeyList.size(); i++) { - GroupKey key = groupKeyList.get(i); - DataFileGroupInternal group = Futures.getDone(groupReadFutures.get(i)); - if (group == null) { - continue; - } - retrievedGroups.add(Pair.create(key, group)); - } - return Futures.immediateFuture(retrievedGroups); - }, - sequentialControlExecutor); - }, - sequentialControlExecutor); + getAllGroupKeys(), + groupKeyList -> { + List<ListenableFuture<@NullableType DataFileGroupInternal>> groupReadFutures = + new ArrayList<>(); + for (GroupKey key : groupKeyList) { + groupReadFutures.add(read(key)); + } + return Futures.whenAllComplete(groupReadFutures) + .callAsync( + () -> { + List<Pair<GroupKey, DataFileGroupInternal>> retrievedGroups = new ArrayList<>(); + for (int i = 0; i < groupKeyList.size(); i++) { + GroupKey key = groupKeyList.get(i); + DataFileGroupInternal group = Futures.getDone(groupReadFutures.get(i)); + if (group == null) { + continue; + } + retrievedGroups.add(Pair.create(key, group)); + } + return Futures.immediateFuture(retrievedGroups); + }, + sequentialControlExecutor); + }, + sequentialControlExecutor); } @Override public ListenableFuture<Boolean> removeAllGroupsWithKeys(List<GroupKey> keys) { SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); + SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); SharedPreferences.Editor editor = prefs.edit(); for (GroupKey key : keys) { LogUtil.d("%s: Removing group %s %s", TAG, key.getGroupName(), key.getOwnerPackage()); @@ -216,8 +217,8 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta @Override public ListenableFuture<List<DataFileGroupInternal>> getAllStaleGroups() { return Futures.immediateFuture( - FileGroupsMetadataUtil.getAllStaleGroups( - FileGroupsMetadataUtil.getGarbageCollectorFile(context, instanceId))); + FileGroupsMetadataUtil.getAllStaleGroups( + FileGroupsMetadataUtil.getGarbageCollectorFile(context, instanceId))); } @Override @@ -226,8 +227,8 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta long currentTimeSeconds = timeSource.currentTimeMillis() / 1000; fileGroup = - FileGroupUtil.setStaleExpirationDate( - fileGroup, currentTimeSeconds + fileGroup.getStaleLifetimeSecs()); + FileGroupUtil.setStaleExpirationDate( + fileGroup, currentTimeSeconds + fileGroup.getStaleLifetimeSecs()); List<DataFileGroupInternal> fileGroups = new ArrayList<>(); fileGroups.add(fileGroup); @@ -275,12 +276,12 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta @Override public ListenableFuture<Void> clear() { SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); + SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); prefs.edit().clear().commit(); SharedPreferences activatedGroupPrefs = - SharedPreferencesUtil.getSharedPreferences( - context, MDD_FILE_GROUP_KEY_PROPERTIES, instanceId); + SharedPreferencesUtil.getSharedPreferences( + context, MDD_FILE_GROUP_KEY_PROPERTIES, instanceId); activatedGroupPrefs.edit().clear().commit(); return removeAllStaleGroups(); diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/dagger/MainMddLibModule.java b/java/com/google/android/libraries/mobiledatadownload/internal/dagger/MainMddLibModule.java index 18c23f0..04ab285 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/dagger/MainMddLibModule.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/dagger/MainMddLibModule.java @@ -16,6 +16,7 @@ package com.google.android.libraries.mobiledatadownload.internal.dagger; import android.content.Context; + import com.google.android.libraries.mobiledatadownload.AccountSource; import com.google.android.libraries.mobiledatadownload.ExperimentationConfig; import com.google.android.libraries.mobiledatadownload.Flags; @@ -33,156 +34,164 @@ import com.google.android.libraries.mobiledatadownload.internal.experimentation. import com.google.android.libraries.mobiledatadownload.internal.experimentation.NoOpDownloadStageManager; import com.google.android.libraries.mobiledatadownload.internal.logging.EventLogger; import com.google.android.libraries.mobiledatadownload.internal.logging.LoggingStateStore; -import com.google.android.libraries.mobiledatadownload.internal.logging.NoOpLoggingState; +import com.google.android.libraries.mobiledatadownload.internal.logging.SharedPreferencesLoggingState; import com.google.android.libraries.mobiledatadownload.internal.util.FuturesUtil; import com.google.android.libraries.mobiledatadownload.monitor.DownloadProgressMonitor; import com.google.android.libraries.mobiledatadownload.monitor.NetworkUsageMonitor; import com.google.common.base.Optional; -import dagger.Module; -import dagger.Provides; + +import java.security.SecureRandom; import java.util.concurrent.Executor; + import javax.inject.Singleton; +import dagger.Module; +import dagger.Provides; + /** Module for MDD Lib dependencies */ @Module public class MainMddLibModule { - /** The version of MDD library. Same as mdi_download module version. */ - // TODO(b/122271766): Figure out how to update this automatically. - // LINT.IfChange - public static final int MDD_LIB_VERSION = 422883838; - // LINT.ThenChange(<internal>) - - private final SynchronousFileStorage fileStorage; - private final NetworkUsageMonitor networkUsageMonitor; - private final EventLogger eventLogger; - private final Optional<DownloadProgressMonitor> downloadProgressMonitorOptional; - private final Optional<SilentFeedback> silentFeedbackOptional; - private final Optional<String> instanceId; - private final Optional<AccountSource> accountSourceOptional; - private final Flags flags; - private final Optional<ExperimentationConfig> experimentationConfigOptional; - - public MainMddLibModule( - SynchronousFileStorage fileStorage, - NetworkUsageMonitor networkUsageMonitor, - EventLogger eventLogger, - Optional<DownloadProgressMonitor> downloadProgressMonitorOptional, - Optional<SilentFeedback> silentFeedbackOptional, - Optional<String> instanceId, - Optional<AccountSource> accountSourceOptional, - Flags flags, - Optional<ExperimentationConfig> experimentationConfigOptional) { - this.fileStorage = fileStorage; - this.networkUsageMonitor = networkUsageMonitor; - this.eventLogger = eventLogger; - this.downloadProgressMonitorOptional = downloadProgressMonitorOptional; - this.silentFeedbackOptional = silentFeedbackOptional; - this.instanceId = instanceId; - this.accountSourceOptional = accountSourceOptional; - this.flags = flags; - this.experimentationConfigOptional = experimentationConfigOptional; - } - - @Provides - @Singleton - static FileGroupsMetadata provideFileGroupsMetadata( - SharedPreferencesFileGroupsMetadata fileGroupsMetadata) { - return fileGroupsMetadata; - } - - @Provides - @Singleton - static SharedFilesMetadata provideSharedFilesMetadata( - SharedPreferencesSharedFilesMetadata sharedFilesMetadata) { - return sharedFilesMetadata; - } - - @Provides - @Singleton - EventLogger provideEventLogger() { - return eventLogger; - } - - @Provides - @Singleton - SilentFeedback providesSilentFeedback() { - if (this.silentFeedbackOptional.isPresent()) { - return this.silentFeedbackOptional.get(); - } else { - return (throwable, description, args) -> { - // No-op SilentFeedback. - }; - } - } - - @Provides - @Singleton - Optional<AccountSource> provideAccountSourceOptional(@ApplicationContext Context context) { - return this.accountSourceOptional; - } - - @Provides - @Singleton - static TimeSource provideTimeSource() { - return System::currentTimeMillis; - } - - @Provides - @Singleton - @InstanceId - Optional<String> provideInstanceId() { - return this.instanceId; - } - - @Provides - @Singleton - NetworkUsageMonitor provideNetworkUsageMonitor() { - return this.networkUsageMonitor; - } - - @Provides - @Singleton - // TODO: We don't need to have @Singleton here and few other places in this class - // since it comes from the this instance. We should remove this since it could increase APK size. - Optional<DownloadProgressMonitor> provideDownloadProgressMonitor() { - return this.downloadProgressMonitorOptional; - } - - @Provides - @Singleton - SynchronousFileStorage provideSynchronousFileStorage() { - return this.fileStorage; - } - - @Provides - @Singleton - Flags provideFlags() { - return this.flags; - } - - @Provides - Optional<ExperimentationConfig> provideExperimentationConfigOptional() { - return this.experimentationConfigOptional; - } - - @Provides - @Singleton - static FuturesUtil provideFuturesUtil(@SequentialControlExecutor Executor sequentialExecutor) { - return new FuturesUtil(sequentialExecutor); - } - - @Provides - @Singleton - static LoggingStateStore provideLoggingStateStore() { - return new NoOpLoggingState(); - } - - @Provides - static DownloadStageManager provideDownloadStageManager( - FileGroupsMetadata fileGroupsMetadata, - Optional<ExperimentationConfig> experimentationConfigOptional, - @SequentialControlExecutor Executor executor, - Flags flags) { - return new NoOpDownloadStageManager(); - } + /** The version of MDD library. Same as mdi_download module version. */ + // TODO(b/122271766): Figure out how to update this automatically. + public static final int MDD_LIB_VERSION = 422883838; + + private final SynchronousFileStorage fileStorage; + private final NetworkUsageMonitor networkUsageMonitor; + private final EventLogger eventLogger; + private final Optional<DownloadProgressMonitor> downloadProgressMonitorOptional; + private final Optional<SilentFeedback> silentFeedbackOptional; + private final Optional<String> instanceId; + private final Optional<AccountSource> accountSourceOptional; + private final Flags flags; + private final Optional<ExperimentationConfig> experimentationConfigOptional; + + public MainMddLibModule( + SynchronousFileStorage fileStorage, + NetworkUsageMonitor networkUsageMonitor, + EventLogger eventLogger, + Optional<DownloadProgressMonitor> downloadProgressMonitorOptional, + Optional<SilentFeedback> silentFeedbackOptional, + Optional<String> instanceId, + Optional<AccountSource> accountSourceOptional, + Flags flags, + Optional<ExperimentationConfig> experimentationConfigOptional) { + this.fileStorage = fileStorage; + this.networkUsageMonitor = networkUsageMonitor; + this.eventLogger = eventLogger; + this.downloadProgressMonitorOptional = downloadProgressMonitorOptional; + this.silentFeedbackOptional = silentFeedbackOptional; + this.instanceId = instanceId; + this.accountSourceOptional = accountSourceOptional; + this.flags = flags; + this.experimentationConfigOptional = experimentationConfigOptional; + } + + @Provides + @Singleton + static FileGroupsMetadata provideFileGroupsMetadata( + SharedPreferencesFileGroupsMetadata fileGroupsMetadata) { + return fileGroupsMetadata; + } + + @Provides + @Singleton + static SharedFilesMetadata provideSharedFilesMetadata( + SharedPreferencesSharedFilesMetadata sharedFilesMetadata) { + return sharedFilesMetadata; + } + + @Provides + @Singleton + EventLogger provideEventLogger() { + return eventLogger; + } + + @Provides + @Singleton + SilentFeedback providesSilentFeedback() { + if (this.silentFeedbackOptional.isPresent()) { + return this.silentFeedbackOptional.get(); + } else { + return (throwable, description, args) -> { + // No-op SilentFeedback. + }; + } + } + + @Provides + @Singleton + Optional<AccountSource> provideAccountSourceOptional(@ApplicationContext Context context) { + return this.accountSourceOptional; + } + + @Provides + @Singleton + static TimeSource provideTimeSource() { + return System::currentTimeMillis; + } + + @Provides + @Singleton + @InstanceId + Optional<String> provideInstanceId() { + return this.instanceId; + } + + @Provides + @Singleton + NetworkUsageMonitor provideNetworkUsageMonitor() { + return this.networkUsageMonitor; + } + + @Provides + @Singleton + // TODO(b/243706147): We don't need to have @Singleton here and few other places in this + // class since it comes from the this instance. We should remove this since it could + // increase APK size. + Optional<DownloadProgressMonitor> provideDownloadProgressMonitor() { + return this.downloadProgressMonitorOptional; + } + + @Provides + @Singleton + SynchronousFileStorage provideSynchronousFileStorage() { + return this.fileStorage; + } + + @Provides + @Singleton + Flags provideFlags() { + return this.flags; + } + + @Provides + Optional<ExperimentationConfig> provideExperimentationConfigOptional() { + return this.experimentationConfigOptional; + } + + @Provides + @Singleton + static FuturesUtil provideFuturesUtil(@SequentialControlExecutor Executor sequentialExecutor) { + return new FuturesUtil(sequentialExecutor); + } + + @Provides + @Singleton + static LoggingStateStore provideLoggingStateStore( + @ApplicationContext Context context, + @InstanceId Optional<String> instanceId, + TimeSource timeSource, + @SequentialControlExecutor Executor sequentialExecutor) { + return SharedPreferencesLoggingState.createFromContext( + context, instanceId, timeSource, sequentialExecutor, new SecureRandom()); + } + + @Provides + static DownloadStageManager provideDownloadStageManager( + FileGroupsMetadata fileGroupsMetadata, + Optional<ExperimentationConfig> experimentationConfigOptional, + @SequentialControlExecutor Executor executor, + Flags flags) { + return new NoOpDownloadStageManager(); + } } diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/EventLogger.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/EventLogger.java index e1ed276..8271ac3 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/EventLogger.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/EventLogger.java @@ -18,6 +18,8 @@ package com.google.android.libraries.mobiledatadownload.internal.logging; import com.google.auto.value.AutoValue; import com.google.common.util.concurrent.AsyncCallable; import com.google.common.util.concurrent.ListenableFuture; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; +import com.google.mobiledatadownload.LogProto.MddFileGroupStatus; import java.util.List; /** Interface for remote logging. */ @@ -28,11 +30,11 @@ public interface EventLogger { /** Log an mdd event with an associated file group. */ void logEventSampled( - int eventCode, - String fileGroupName, - int fileGroupVersionNumber, - long buildId, - String variantId); + int eventCode, + String fileGroupName, + int fileGroupVersionNumber, + long buildId, + String variantId); /** * Log an mdd event. This not sampled. Caller should make sure this method is called after @@ -50,18 +52,19 @@ public interface EventLogger { * failure if the callable fails or if there is an error when logging. */ ListenableFuture<Void> logMddFileGroupStats( - AsyncCallable<List<FileGroupStatusWithDetails>> buildFileGroupStats); + AsyncCallable<List<FileGroupStatusWithDetails>> buildFileGroupStats); /** Simple wrapper class for MDD file group stats and details. */ @AutoValue abstract class FileGroupStatusWithDetails { - abstract Void fileGroupStatus(); + abstract MddFileGroupStatus fileGroupStatus(); - abstract Void fileGroupDetails(); + abstract DataDownloadFileGroupStats fileGroupDetails(); - static FileGroupStatusWithDetails create(Void fileGroupStatus, Void fileGroupDetails) { + static FileGroupStatusWithDetails create( + MddFileGroupStatus fileGroupStatus, DataDownloadFileGroupStats fileGroupDetails) { return new AutoValue_EventLogger_FileGroupStatusWithDetails( - fileGroupStatus, fileGroupDetails); + fileGroupStatus, fileGroupDetails); } } @@ -93,12 +96,12 @@ public interface EventLogger { /** Log the network savings of MDD download features */ void logMddNetworkSavings( - Void fileGroupDetails, - int code, - long fullFileSize, - long downloadedFileSize, - String fileId, - int deltaIndex); + Void fileGroupDetails, + int code, + long fullFileSize, + long downloadedFileSize, + String fileId, + int deltaIndex); /** Log mdd download result events. */ void logMddDownloadResult(int code, Void fileGroupDetails); @@ -114,4 +117,4 @@ public interface EventLogger { /** Log mdd usage event. */ void logMddUsageEvent(Void fileGroupDetails, Void usageEventLog); -} +}
\ No newline at end of file diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/FileGroupStatsLogger.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/FileGroupStatsLogger.java index 3803c33..6ef267b 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/FileGroupStatsLogger.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/FileGroupStatsLogger.java @@ -15,15 +15,22 @@ */ package com.google.android.libraries.mobiledatadownload.internal.logging; +import static com.google.common.util.concurrent.Futures.immediateFuture; + import android.util.Pair; import com.google.android.libraries.mobiledatadownload.internal.FileGroupManager; +import com.google.android.libraries.mobiledatadownload.internal.FileGroupManager.GroupDownloadStatus; import com.google.android.libraries.mobiledatadownload.internal.FileGroupsMetadata; import com.google.android.libraries.mobiledatadownload.internal.annotations.SequentialControlExecutor; +import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupUtil; import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; +import com.google.mobiledatadownload.LogEnumsProto.MddFileGroupDownloadStatus; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; +import com.google.mobiledatadownload.LogProto.MddFileGroupStatus; import java.util.ArrayList; import java.util.List; import java.util.concurrent.Executor; @@ -43,10 +50,10 @@ public class FileGroupStatsLogger { @Inject public FileGroupStatsLogger( - FileGroupManager fileGroupManager, - FileGroupsMetadata fileGroupsMetadata, - EventLogger eventLogger, - @SequentialControlExecutor Executor sequentialControlExecutor) { + FileGroupManager fileGroupManager, + FileGroupsMetadata fileGroupsMetadata, + EventLogger eventLogger, + @SequentialControlExecutor Executor sequentialControlExecutor) { this.fileGroupManager = fileGroupManager; this.fileGroupsMetadata = fileGroupsMetadata; this.eventLogger = eventLogger; @@ -59,36 +66,80 @@ public class FileGroupStatsLogger { } private ListenableFuture<List<EventLogger.FileGroupStatusWithDetails>> buildFileGroupStatusList( - int daysSinceLastLog) { + int daysSinceLastLog) { return PropagatedFutures.transformAsync( - fileGroupsMetadata.getAllFreshGroups(), - downloadedAndPendingGroups -> { - List<ListenableFuture<EventLogger.FileGroupStatusWithDetails>> futures = - new ArrayList<>(); - for (Pair<GroupKey, DataFileGroupInternal> pair : downloadedAndPendingGroups) { - GroupKey groupKey = pair.first; - DataFileGroupInternal dataFileGroup = pair.second; - if (dataFileGroup == null) { - continue; - } + fileGroupsMetadata.getAllFreshGroups(), + downloadedAndPendingGroups -> { + List<ListenableFuture<EventLogger.FileGroupStatusWithDetails>> futures = + new ArrayList<>(); + for (Pair<GroupKey, DataFileGroupInternal> pair : downloadedAndPendingGroups) { + GroupKey groupKey = pair.first; + DataFileGroupInternal dataFileGroup = pair.second; + if (dataFileGroup == null) { + continue; + } - Void fileGroupDetails = null; + DataDownloadFileGroupStats fileGroupDetails = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(groupKey.getGroupName()) + .setOwnerPackage(groupKey.getOwnerPackage()) + .setFileGroupVersionNumber(dataFileGroup.getFileGroupVersionNumber()) + .setFileCount(dataFileGroup.getFileCount()) + .setInlineFileCount(FileGroupUtil.getInlineFileCount(dataFileGroup)) + .setHasAccount(!groupKey.getAccount().isEmpty()) + .setBuildId(dataFileGroup.getBuildId()) + .setVariantId(dataFileGroup.getVariantId()) + .build(); - futures.add( - PropagatedFutures.transform( - buildFileGroupStatus(dataFileGroup, groupKey, daysSinceLastLog), - fileGroupStatus -> - EventLogger.FileGroupStatusWithDetails.create( - fileGroupStatus, fileGroupDetails), - sequentialControlExecutor)); - } - return Futures.allAsList(futures); - }, - sequentialControlExecutor); + futures.add( + PropagatedFutures.transform( + buildFileGroupStatus(dataFileGroup, groupKey, daysSinceLastLog), + fileGroupStatus -> + EventLogger.FileGroupStatusWithDetails.create( + fileGroupStatus, fileGroupDetails), + sequentialControlExecutor)); + } + return Futures.allAsList(futures); + }, + sequentialControlExecutor); } - private ListenableFuture<Void> buildFileGroupStatus( - DataFileGroupInternal dataFileGroup, GroupKey groupKey, int daysSinceLastLog) { - return Futures.immediateVoidFuture(); + private ListenableFuture<MddFileGroupStatus> buildFileGroupStatus( + DataFileGroupInternal dataFileGroup, GroupKey groupKey, int daysSinceLastLog) { + MddFileGroupStatus.Builder fileGroupStatus = + MddFileGroupStatus.newBuilder().setDaysSinceLastLog(daysSinceLastLog); + if (dataFileGroup.getBookkeeping().hasGroupNewFilesReceivedTimestamp()) { + fileGroupStatus.setGroupAddedTimestampInSeconds( + dataFileGroup.getBookkeeping().getGroupNewFilesReceivedTimestamp() / 1000); + } else { + fileGroupStatus.setGroupAddedTimestampInSeconds(-1); + } + + if (groupKey.getDownloaded()) { + fileGroupStatus.setFileGroupDownloadStatus(MddFileGroupDownloadStatus.Code.COMPLETE); + if (dataFileGroup.getBookkeeping().hasGroupDownloadedTimestampInMillis()) { + fileGroupStatus.setGroupDownloadedTimestampInSeconds( + dataFileGroup.getBookkeeping().getGroupDownloadedTimestampInMillis() / 1000); + } else { + fileGroupStatus.setGroupDownloadedTimestampInSeconds(-1); + } + return immediateFuture(fileGroupStatus.build()); + } else { + fileGroupStatus.setGroupDownloadedTimestampInSeconds(-1); + return PropagatedFutures.transform( + fileGroupManager.getFileGroupDownloadStatus(dataFileGroup), + status -> { + if (status == GroupDownloadStatus.DOWNLOADED || status == GroupDownloadStatus.PENDING) { + // Log pending even if verify returns downloaded, as it will be marked as + // completed in the next periodic task. + fileGroupStatus.setFileGroupDownloadStatus(MddFileGroupDownloadStatus.Code.PENDING); + } else { + // TODO(b/73490689): Log the reason for failure along with this. + fileGroupStatus.setFileGroupDownloadStatus(MddFileGroupDownloadStatus.Code.FAILED); + } + return fileGroupStatus.build(); + }, + sequentialControlExecutor); + } } -} +}
\ No newline at end of file diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/LogSampler.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/LogSampler.java index 212dec5..6e6ac72 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/LogSampler.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/LogSampler.java @@ -23,6 +23,9 @@ import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentF import com.google.common.base.Optional; import com.google.common.util.concurrent.ListenableFuture; import com.google.errorprone.annotations.CheckReturnValue; +import com.google.mobiledatadownload.LogProto.StableSamplingInfo; +import com.google.protobuf.Timestamp; + import java.util.Random; /** Class responsible for sampling events. */ @@ -58,8 +61,8 @@ public final class LogSampler { * Optional if the event should not be logged. If the event should be logged, the returned * Void should be attached to the log event. */ - public ListenableFuture<Optional<Void>> shouldLog( - long sampleInterval, Optional<LoggingStateStore> loggingStateStore) { + public ListenableFuture<Optional<StableSamplingInfo>> shouldLog( + long sampleInterval, Optional<LoggingStateStore> loggingStateStore) { if (sampleInterval == 0L) { return immediateFuture(Optional.absent()); } else if (sampleInterval < 0L) { @@ -78,9 +81,10 @@ public final class LogSampler { * @return if the event should be sampled, returns the Void with stable_sampling_used = false. * Otherwise, returns an empty Optional. */ - private ListenableFuture<Optional<Void>> shouldLogPerEvent(long sampleInterval) { + private ListenableFuture<Optional<StableSamplingInfo>> shouldLogPerEvent(long sampleInterval) { if (shouldSamplePerEvent(sampleInterval)) { - return immediateFuture(Optional.absent()); + return immediateFuture( + Optional.of(StableSamplingInfo.newBuilder().setStableSamplingUsed(false).build())); } else { return immediateFuture(Optional.absent()); } @@ -103,24 +107,33 @@ public final class LogSampler { * @return if the event should be sampled, returns the Void with stable_sampling_used = true and * all other fields populated. Otherwise, returns an empty Optional. */ - private ListenableFuture<Optional<Void>> shouldLogDeviceStable( - long sampleInterval, LoggingStateStore loggingStateStore) { + private ListenableFuture<Optional<StableSamplingInfo>> shouldLogDeviceStable( + long sampleInterval, LoggingStateStore loggingStateStore) { return PropagatedFluentFuture.from(loggingStateStore.getStableSamplingInfo()) - .transform( - samplingInfo -> { - boolean invalidSamplingRateUsed = ((100 % sampleInterval) != 0); - if (invalidSamplingRateUsed) { - LogUtil.e( - "Bad sample interval (1 percent cohort will not log): %d", sampleInterval); - } + .transform( + samplingInfo -> { + boolean invalidSamplingRateUsed = ((100 % sampleInterval) != 0); + if (invalidSamplingRateUsed) { + LogUtil.e( + "Bad sample interval (1 percent cohort will not log): %d", sampleInterval); + } - if (!isPartOfSample(samplingInfo.getStableLogSamplingSalt(), sampleInterval)) { - return Optional.absent(); - } + if (!isPartOfSample(samplingInfo.getStableLogSamplingSalt(), sampleInterval)) { + return Optional.absent(); + } - return Optional.absent(); - }, - directExecutor()); + return Optional.of( + StableSamplingInfo.newBuilder() + .setStableSamplingUsed(true) + .setStableSamplingFirstEnabledTimestampMs( + toMillis(samplingInfo.getLogSamplingSaltSetTimestamp())) + .setPartOfAlwaysLoggingGroup( + isPartOfSample( + samplingInfo.getStableLogSamplingSalt(), /*sampleInterval=*/ 100)) + .setInvalidSamplingRateUsed(invalidSamplingRateUsed) + .build()); + }, + directExecutor()); } /** @@ -130,4 +143,10 @@ public final class LogSampler { private boolean isPartOfSample(long randomNumber, long sampleInterval) { return randomNumber % sampleInterval == 0; } -} + + // Copy from com.google.protobuf.util.Timestamps + // TODO(b/243397277) Remove toMillis. + private static long toMillis(Timestamp timestamp) { + return timestamp.getSeconds() * 1000L + (long)timestamp.getNanos() / 1000000L; + } +}
\ No newline at end of file diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/MddEventLogger.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/MddEventLogger.java index c2b4984..1b11bb4 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/MddEventLogger.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/MddEventLogger.java @@ -29,6 +29,11 @@ import com.google.common.util.concurrent.AsyncCallable; import com.google.common.util.concurrent.FluentFuture; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.ListenableFuture; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent.Code; +import com.google.mobiledatadownload.LogProto.AndroidClientInfo; +import com.google.mobiledatadownload.LogProto.MddDeviceInfo; +import com.google.mobiledatadownload.LogProto.MddLogData; +import com.google.mobiledatadownload.LogProto.StableSamplingInfo; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -50,7 +55,7 @@ public final class MddEventLogger implements EventLogger { private Optional<LoggingStateStore> loggingStateStore = Optional.absent(); public MddEventLogger( - Context context, Logger logger, int moduleVersion, LogSampler logSampler, Flags flags) { + Context context, Logger logger, int moduleVersion, LogSampler logSampler, Flags flags) { this.context = context; this.logger = logger; this.moduleVersion = moduleVersion; @@ -78,11 +83,11 @@ public final class MddEventLogger implements EventLogger { @Override public void logEventSampled( - int eventCode, - String fileGroupName, - int fileGroupVersionNumber, - long buildId, - String variantId) { + int eventCode, + String fileGroupName, + int fileGroupVersionNumber, + long buildId, + String variantId) { Void dataDownloadFileGroupStats = null; } @@ -110,43 +115,48 @@ public final class MddEventLogger implements EventLogger { @Override public ListenableFuture<Void> logMddFileGroupStats( - AsyncCallable<List<EventLogger.FileGroupStatusWithDetails>> buildFileGroupStats) { + AsyncCallable<List<EventLogger.FileGroupStatusWithDetails>> buildFileGroupStats) { return lazySampleAndSendLogEvent( - 0, - () -> - PropagatedFutures.transform( - buildFileGroupStats.call(), - fileGroupStatusAndDetailsList -> { - List<Void> allIcingLogData = new ArrayList<>(); - - for (FileGroupStatusWithDetails fileGroupStatusAndDetails : - fileGroupStatusAndDetailsList) { - allIcingLogData.add(null); - } - return allIcingLogData; - }, - directExecutor()), - flags.groupStatsLoggingSampleInterval()); + Code.DATA_DOWNLOAD_FILE_GROUP_STATUS, + () -> + PropagatedFutures.transform( + buildFileGroupStats.call(), + fileGroupStatusAndDetailsList -> { + List<MddLogData> allMddLogData = new ArrayList<>(); + + for (FileGroupStatusWithDetails fileGroupStatusAndDetails : + fileGroupStatusAndDetailsList) { + allMddLogData.add( + MddLogData.newBuilder() + .setMddFileGroupStatus(fileGroupStatusAndDetails.fileGroupStatus()) + .setDataDownloadFileGroupStats( + fileGroupStatusAndDetails.fileGroupDetails()) + .build()); + } + return allMddLogData; + }, + directExecutor()), + flags.groupStatsLoggingSampleInterval()); } @Override public ListenableFuture<Void> logMddStorageStats(AsyncCallable<Void> buildStorageStats) { return lazySampleAndSendLogEvent( - 0, - () -> - PropagatedFutures.transform( - buildStorageStats.call(), storageStats -> Arrays.asList(), directExecutor()), - flags.storageStatsLoggingSampleInterval()); + Code.EVENT_CODE_UNSPECIFIED, + () -> + PropagatedFutures.transform( + buildStorageStats.call(), storageStats -> Arrays.asList(), directExecutor()), + flags.storageStatsLoggingSampleInterval()); } @Override public ListenableFuture<Void> logMddNetworkStats(AsyncCallable<Void> buildNetworkStats) { return lazySampleAndSendLogEvent( - 0, - () -> - PropagatedFutures.transform( - buildNetworkStats.call(), networkStats -> Arrays.asList(), directExecutor()), - flags.networkStatsLoggingSampleInterval()); + Code.EVENT_CODE_UNSPECIFIED, + () -> + PropagatedFutures.transform( + buildNetworkStats.call(), networkStats -> Arrays.asList(), directExecutor()), + flags.networkStatsLoggingSampleInterval()); } @Override @@ -157,12 +167,12 @@ public final class MddEventLogger implements EventLogger { @Override public void logMddNetworkSavings( - Void fileGroupDetails, - int code, - long fullFileSize, - long downloadedFileSize, - String fileId, - int deltaIndex) { + Void fileGroupDetails, + int code, + long fullFileSize, + long downloadedFileSize, + String fileId, + int deltaIndex) { Void logData = null; sampleAndSendLogEvent(0, logData, flags.mddDefaultSampleInterval()); @@ -213,63 +223,92 @@ public final class MddEventLogger implements EventLogger { * constructs the log event lazy. This is useful if constructing the log event is expensive. */ private ListenableFuture<Void> lazySampleAndSendLogEvent( - int eventCode, AsyncCallable<List<Void>> buildStats, int sampleInterval) { + Code eventCode, AsyncCallable<List<MddLogData>> buildStats, int sampleInterval) { return PropagatedFutures.transformAsync( - logSampler.shouldLog(sampleInterval, loggingStateStore), - samplingInfoOptional -> { - if (!samplingInfoOptional.isPresent()) { - return immediateVoidFuture(); - } - - return FluentFuture.from(buildStats.call()) - .transform( - icingLogDataList -> { - if (icingLogDataList != null) { - for (Void icingLogData : icingLogDataList) { - processAndSendEvent( - eventCode, null, sampleInterval, samplingInfoOptional.get()); - } - } - return null; - }, - directExecutor()); - }, - directExecutor()); + logSampler.shouldLog(sampleInterval, loggingStateStore), + samplingInfoOptional -> { + if (!samplingInfoOptional.isPresent()) { + return immediateVoidFuture(); + } + + return FluentFuture.from(buildStats.call()) + .transform( + icingLogDataList -> { + if (icingLogDataList != null) { + for (MddLogData icingLogData : icingLogDataList) { + processAndSendEvent( + eventCode, + icingLogData.toBuilder(), + sampleInterval, + samplingInfoOptional.get()); + } + } + return null; + }, + directExecutor()); + }, + directExecutor()); } private void sampleAndSendLogEvent(int eventCode, Void logData, long sampleInterval) { PropagatedFutures.addCallback( - logSampler.shouldLog(sampleInterval, loggingStateStore), - new FutureCallback<Optional<Void>>() { - @Override - public void onSuccess(Optional<Void> stableSamplingInfo) { - if (stableSamplingInfo.isPresent()) { - processAndSendEvent(eventCode, logData, sampleInterval, stableSamplingInfo.get()); - } - } - - @Override - public void onFailure(Throwable t) { - LogUtil.e(t, "%s: failure when sampling log!", TAG); - } - }, - directExecutor()); + logSampler.shouldLog(sampleInterval, loggingStateStore), + new FutureCallback<Optional<StableSamplingInfo>>() { + @Override + public void onSuccess(Optional<StableSamplingInfo> stableSamplingInfo) { + if (stableSamplingInfo.isPresent()) { + processAndSendEvent( + Code.EVENT_CODE_UNSPECIFIED, + MddLogData.newBuilder(), + sampleInterval, + stableSamplingInfo.get()); + } + } + + @Override + public void onFailure(Throwable t) { + LogUtil.e(t, "%s: failure when sampling log!", TAG); + } + }, + directExecutor()); } /** Adds all transforms common to all logs and sends the event to Logger. */ private void processAndSendEventWithoutStableSampling( - int eventCode, Void logData, long sampleInterval) { - processAndSendEvent(eventCode, logData, sampleInterval, null); + int eventCode, Void logData, long sampleInterval) { + processAndSendEvent( + Code.EVENT_CODE_UNSPECIFIED, + MddLogData.newBuilder(), + sampleInterval, + StableSamplingInfo.newBuilder().setStableSamplingUsed(false).build()); } /** Adds all transforms common to all logs and sends the event to Logger. */ private void processAndSendEvent( - int eventCode, Void logData, long sampleInterval, Void stableSamplingInfo) {} + Code eventCode, + MddLogData.Builder logData, + long sampleInterval, + StableSamplingInfo stableSamplingInfo) { + if (eventCode.equals(Code.EVENT_CODE_UNSPECIFIED)) { + LogUtil.e("%s: unspecified code used, skipping event log", TAG); + // return early for unspecified codes. + return; + } + logData + .setSamplingInterval(sampleInterval) + .setDeviceInfo(MddDeviceInfo.newBuilder().setDeviceStorageLow(isDeviceStorageLow(context))) + .setAndroidClientInfo( + AndroidClientInfo.newBuilder() + .setHostPackageName(hostPackageName) + .setModuleVersion(moduleVersion)) + .setStableSamplingInfo(stableSamplingInfo); + logger.log(logData.build(), eventCode.getNumber()); + } /** Returns whether the device is in low storage state. */ private static boolean isDeviceStorageLow(Context context) { // Check if the system says storage is low, by reading the sticky intent. return context.registerReceiver(null, new IntentFilter(Intent.ACTION_DEVICE_STORAGE_LOW)) - != null; + != null; } -} +}
\ No newline at end of file diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/SharedPreferencesLoggingState.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/SharedPreferencesLoggingState.java new file mode 100644 index 0000000..e4debc5 --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/SharedPreferencesLoggingState.java @@ -0,0 +1,367 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.libraries.mobiledatadownload.internal.logging; + +import static com.google.android.libraries.mobiledatadownload.internal.MddConstants.SPLIT_CHAR; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +import android.content.Context; +import android.content.SharedPreferences; +import androidx.annotation.VisibleForTesting; +import com.google.android.libraries.mobiledatadownload.TimeSource; +import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupsMetadataUtil; +import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupsMetadataUtil.GroupKeyDeserializationException; +import com.google.android.libraries.mobiledatadownload.internal.util.SharedPreferencesUtil; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedExecutionSequencer; +import com.google.common.base.Optional; +import com.google.common.base.Splitter; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import com.google.common.primitives.Ints; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.mobiledatadownload.internal.MetadataProto.FileGroupLoggingState; +import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; +import com.google.mobiledatadownload.internal.MetadataProto.SamplingInfo; +import com.google.protobuf.Timestamp; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.GregorianCalendar; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.TimeZone; +import java.util.concurrent.Executor; + +/** LoggingStateStore that uses SharedPreferences for storage. */ +public final class SharedPreferencesLoggingState implements LoggingStateStore { + + private static final String SHARED_PREFS_NAME = "LoggingState"; + + private static final String LAST_MAINTENANCE_RUN_SECS_KEY = "last_maintenance_secs"; + + @VisibleForTesting static final String SALT_KEY = "stable_log_sampling_salt"; + private static final String SALT_TIMESTAMP_MILLIS_KEY = "log_sampling_salt_set_timestamp_millis"; + + private final Supplier<SharedPreferences> sharedPrefs; + private final Executor backgroundExecutor; + private final TimeSource timeSource; + private final Random random; + + // Serialize access to SharedPref keys to avoid clobbering. + private final PropagatedExecutionSequencer futureSerializer = + PropagatedExecutionSequencer.create(); + + /** + * Constructs a new instance. + * + * @param sharedPrefs may be called multiple times, so memoization is recommended. The returned + * instance must be exclusive to {@link SharedPreferencesLoggingState} since {@link #clear} + * may clear the data at any time. + */ + public static SharedPreferencesLoggingState create( + Supplier<SharedPreferences> sharedPrefs, + TimeSource timeSource, + Executor backgroundExecutor, + Random random) { + return new SharedPreferencesLoggingState(sharedPrefs, timeSource, backgroundExecutor, random); + } + + /** Constructs a new instance. */ + public static SharedPreferencesLoggingState createFromContext( + Context context, + Optional<String> instanceIdOptional, + TimeSource timeSource, + Executor backgroundExecutor, + Random random) { + // Avoid calling getSharedPreferences on the main thread. + Supplier<SharedPreferences> sharedPrefs = + Suppliers.memoize( + () -> + SharedPreferencesUtil.getSharedPreferences( + context, SHARED_PREFS_NAME, instanceIdOptional)); + return new SharedPreferencesLoggingState(sharedPrefs, timeSource, backgroundExecutor, random); + } + + private SharedPreferencesLoggingState( + Supplier<SharedPreferences> sharedPrefs, + TimeSource timeSource, + Executor backgroundExecutor, + Random random) { + this.sharedPrefs = sharedPrefs; + this.timeSource = timeSource; + this.backgroundExecutor = backgroundExecutor; + this.random = random; + } + + /** Data fields for each Entry persisted in SharedPreferences. */ + private enum Key { + CELLULAR_USAGE("cu"), + WIFI_USAGE("wu"); + + final String sharedPrefsSuffix; + + Key(String sharedPrefsSuffix) { + this.sharedPrefsSuffix = sharedPrefsSuffix; + } + } + + /** Bridge between FileGroupLoggingState and its SharedPreferences representation. */ + private static final class Entry { + + final GroupKey groupKey; + final long buildId; + final int fileGroupVersionNumber; + + /** Prefix used in SharedPreference keys. */ + final String spKeyPrefix; + + static Entry fromLoggingState(FileGroupLoggingState loggingState) { + return new Entry( + /* groupKey= */ loggingState.getGroupKey(), + /* buildId= */ loggingState.getBuildId(), + /* fileGroupVersionNumber= */ loggingState.getFileGroupVersionNumber()); + } + + /** + * @throws IllegalArgumentException if the key can't be parsed + */ + static Entry fromSpKey(String spKey) { + List<String> parts = Splitter.on(SPLIT_CHAR).splitToList(spKey); + try { + return new Entry( + /* groupKey= */ FileGroupsMetadataUtil.deserializeGroupKey(parts.get(0)), + /* buildId= */ Long.parseLong(parts.get(1)), + /* fileGroupVersionNumber= */ Integer.parseInt(parts.get(2))); + } catch (GroupKeyDeserializationException | ArrayIndexOutOfBoundsException e) { + throw new IllegalArgumentException("Failed to parse SharedPrefs key: " + spKey, e); + } + } + + private Entry(GroupKey groupKey, long buildId, int fileGroupVersionNumber) { + this.groupKey = groupKey; + this.buildId = buildId; + this.fileGroupVersionNumber = fileGroupVersionNumber; + this.spKeyPrefix = + FileGroupsMetadataUtil.getSerializedGroupKey(groupKey) + + SPLIT_CHAR + + buildId + + SPLIT_CHAR + + fileGroupVersionNumber; + } + + String getSharedPrefsKey(Key key) { + return spKeyPrefix + SPLIT_CHAR + key.sharedPrefsSuffix; + } + } + + @Override + public ListenableFuture<Optional<Integer>> getAndResetDaysSinceLastMaintenance() { + return futureSerializer.submit( + () -> { + long currentTimestamp = timeSource.currentTimeMillis(); + + Optional<Integer> daysSinceLastMaintenance; + boolean hasEverDoneMaintenance = + sharedPrefs.get().contains(LAST_MAINTENANCE_RUN_SECS_KEY); + if (hasEverDoneMaintenance) { + long persistedTimestamp = sharedPrefs.get().getLong(LAST_MAINTENANCE_RUN_SECS_KEY, 0); + long currentStartOfDay = truncateTimestampToStartOfDay(currentTimestamp); + long previousStartOfDay = truncateTimestampToStartOfDay(persistedTimestamp); + // Note: ignore MillisTo_Days java optional suggestion because Duration is api + // 26+. + daysSinceLastMaintenance = + Optional.of( + Ints.saturatedCast( + MILLISECONDS.toDays(currentStartOfDay - previousStartOfDay))); + } else { + daysSinceLastMaintenance = Optional.absent(); + } + + SharedPreferences.Editor editor = sharedPrefs.get().edit(); + editor.putLong(LAST_MAINTENANCE_RUN_SECS_KEY, currentTimestamp); + commitOrThrow(editor); + + return daysSinceLastMaintenance; + }, + backgroundExecutor); + } + + @Override + public ListenableFuture<Void> incrementDataUsage(FileGroupLoggingState dataUsageIncrements) { + return futureSerializer.submit( + () -> { + Entry entry = Entry.fromLoggingState(dataUsageIncrements); + + long currentCellarUsage = + sharedPrefs.get().getLong(entry.getSharedPrefsKey(Key.CELLULAR_USAGE), 0); + long currentWifiUsage = + sharedPrefs.get().getLong(entry.getSharedPrefsKey(Key.WIFI_USAGE), 0); + long updatedCellarUsage = currentCellarUsage + dataUsageIncrements.getCellularUsage(); + long updatedWifiUsage = currentWifiUsage + dataUsageIncrements.getWifiUsage(); + + SharedPreferences.Editor editor = sharedPrefs.get().edit(); + editor.putLong(entry.getSharedPrefsKey(Key.CELLULAR_USAGE), updatedCellarUsage); + editor.putLong(entry.getSharedPrefsKey(Key.WIFI_USAGE), updatedWifiUsage); + + return commitOrThrow(editor); + }, + backgroundExecutor); + } + + @Override + public ListenableFuture<List<FileGroupLoggingState>> getAndResetAllDataUsage() { + return futureSerializer.submit( + () -> { + List<FileGroupLoggingState> allLoggingStates = new ArrayList<>(); + Set<String> allLoggingStateKeys = new HashSet<>(); + SharedPreferences.Editor editor = sharedPrefs.get().edit(); + + for (String key : sharedPrefs.get().getAll().keySet()) { + Entry entry; + try { + entry = Entry.fromSpKey(key); + } catch (IllegalArgumentException e) { + continue; // This isn't a LoggingState entry + } + if (allLoggingStateKeys.contains(entry.spKeyPrefix)) { + continue; + } + allLoggingStateKeys.add(entry.spKeyPrefix); + + FileGroupLoggingState loggingState = + FileGroupLoggingState.newBuilder() + .setGroupKey(entry.groupKey) + .setBuildId(entry.buildId) + .setFileGroupVersionNumber(entry.fileGroupVersionNumber) + .setCellularUsage( + sharedPrefs.get().getLong(entry.getSharedPrefsKey(Key.CELLULAR_USAGE), 0)) + .setWifiUsage( + sharedPrefs.get().getLong(entry.getSharedPrefsKey(Key.WIFI_USAGE), 0)) + .build(); + allLoggingStates.add(loggingState); + + editor.remove(entry.getSharedPrefsKey(Key.CELLULAR_USAGE)); + editor.remove(entry.getSharedPrefsKey(Key.WIFI_USAGE)); + } + commitOrThrow(editor); + + return allLoggingStates; + }, + backgroundExecutor); + } + + @Override + public ListenableFuture<Void> clear() { + return futureSerializer.submit( + () -> { + SharedPreferences.Editor editor = sharedPrefs.get().edit(); + editor.clear(); + return commitOrThrow(editor); + }, + backgroundExecutor); + } + + @Override + public ListenableFuture<SamplingInfo> getStableSamplingInfo() { + return futureSerializer.submit( + () -> { + long salt; + long persistedTimestampMillis; + + boolean hasCreatedSalt = sharedPrefs.get().contains(SALT_KEY); + if (hasCreatedSalt) { + salt = sharedPrefs.get().getLong(SALT_KEY, 0); + persistedTimestampMillis = sharedPrefs.get().getLong(SALT_TIMESTAMP_MILLIS_KEY, 0); + } else { + salt = random.nextLong(); + persistedTimestampMillis = timeSource.currentTimeMillis(); + + SharedPreferences.Editor editor = sharedPrefs.get().edit(); + editor.putLong(SALT_KEY, salt); + editor.putLong(SALT_TIMESTAMP_MILLIS_KEY, persistedTimestampMillis); + commitOrThrow(editor); + } + + Timestamp timestamp = fromMillis(persistedTimestampMillis); + return SamplingInfo.newBuilder() + .setStableLogSamplingSalt(salt) + .setLogSamplingSaltSetTimestamp(timestamp) + .build(); + }, + backgroundExecutor); + } + + // Use UTC time zone here so we don't have to worry about time zone change or daylight savings. + private static final TimeZone UTC_TIMEZONE = TimeZone.getTimeZone("UTC"); + + // TODO(b/237533403): extract as shareable code with ProtoDataStoreLoggingState + private static long truncateTimestampToStartOfDay(long timestampMillis) { + // We use the regular java.util.Calendar classes here since neither Joda time nor java.time is + // supported across all client apps. + Calendar cal = new GregorianCalendar(UTC_TIMEZONE); + cal.setTimeInMillis(timestampMillis); + cal.set(Calendar.HOUR_OF_DAY, 0); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + return cal.getTimeInMillis(); + } + + /** Calls {@code editor.commit()} and returns void, or throws IOException if the commit failed. */ + private static Void commitOrThrow(SharedPreferences.Editor editor) throws IOException { + if (!editor.commit()) { + throw new IOException("Failed to commit"); + } + return null; + } + + // TODO(b/243397277) Remove following methods. + public static Timestamp fromMillis(long milliseconds) { + return normalizedTimestamp(milliseconds / 1000L, (int)(milliseconds % 1000L * 1000000L)); + } + + private static Timestamp normalizedTimestamp(long seconds, int nanos) { + if ((long)nanos <= -1000000000L || (long)nanos >= 1000000000L) { + seconds += (long)nanos / 1000000000L; + nanos = (int)((long)nanos % 1000000000L); + } + + if (nanos < 0) { + nanos = (int)((long)nanos + 1000000000L); + --seconds; + } + + checkValid(seconds, nanos); + return Timestamp.newBuilder().setSeconds(seconds).setNanos(nanos).build(); + } + + private static void checkValid(long seconds, int nanos) { + if (!isValid(seconds, (long)nanos)) { + throw new IllegalArgumentException(String.format("Timestamp is not valid. See proto definition for valid values. Seconds (%s) must be in range [-62,135,596,800, +253,402,300,799].Nanos (%s) must be in range [0, +999,999,999].", seconds, nanos)); + } + } + + private static boolean isValid(long seconds, long nanos) { + if (seconds >= -62135596800L && seconds <= 253402300799L) { + return nanos >= 0L && nanos < 1000000000L; + } else { + return false; + } + } +}
\ No newline at end of file diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/util/FileGroupsMetadataUtil.java b/java/com/google/android/libraries/mobiledatadownload/internal/util/FileGroupsMetadataUtil.java index fda3b1e..7853bfa 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/util/FileGroupsMetadataUtil.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/util/FileGroupsMetadataUtil.java @@ -94,7 +94,7 @@ public final class FileGroupsMetadataUtil { } // TODO(b/129702287): Move away from proto based serialization. - public static String getSerializedGroupKey(GroupKey groupKey, Context context) { + public static String getSerializedGroupKey(GroupKey groupKey) { byte[] byteValue = groupKey.toByteArray(); return Base64.encodeToString(byteValue, Base64.NO_PADDING | Base64.NO_WRAP); } diff --git a/java/com/google/android/libraries/mobiledatadownload/tracing/PropagatedExecutionSequencer.java b/java/com/google/android/libraries/mobiledatadownload/tracing/PropagatedExecutionSequencer.java new file mode 100644 index 0000000..c2bfec5 --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/tracing/PropagatedExecutionSequencer.java @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.tracing; + +import com.google.common.util.concurrent.AsyncCallable; +import com.google.common.util.concurrent.ExecutionSequencer; +import com.google.common.util.concurrent.ListenableFuture; +import java.util.concurrent.Callable; +import java.util.concurrent.Executor; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** Wrapper around {@link ExecutionSequencer} with trace propagation. */ +public final class PropagatedExecutionSequencer { + + private final ExecutionSequencer executionSequencer = ExecutionSequencer.create(); + + private PropagatedExecutionSequencer() {} + + /** Creates a new instance. */ + public static PropagatedExecutionSequencer create() { + return new PropagatedExecutionSequencer(); + } + + /** See {@link ExecutionSequencer#submit(Callable, Executor)}. */ + public <T extends @Nullable Object> ListenableFuture<T> submit( + Callable<T> callable, Executor executor) { + return executionSequencer.submit(callable, executor); + } + + /** See {@link ExecutionSequencer#submitAsync(AsyncCallable, Executor)}. */ + public <T extends @Nullable Object> ListenableFuture<T> submitAsync( + AsyncCallable<T> callable, Executor executor) { + return executionSequencer.submitAsync(callable, executor); + } +}
\ No newline at end of file diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/FileGroupStatsLoggerTest.java b/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/FileGroupStatsLoggerTest.java new file mode 100644 index 0000000..ad7777a --- /dev/null +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/FileGroupStatsLoggerTest.java @@ -0,0 +1,339 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.internal.logging; + +import static com.google.common.truth.Truth.assertThat; +import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import android.util.Pair; +import com.google.android.libraries.mdi.download.MetadataProto.DataFile; +import com.google.android.libraries.mdi.download.MetadataProto.DataFileGroupBookkeeping; +import com.google.android.libraries.mdi.download.MetadataProto.DataFileGroupInternal; +import com.google.android.libraries.mdi.download.MetadataProto.GroupKey; +import com.google.android.libraries.mobiledatadownload.internal.FileGroupManager; +import com.google.android.libraries.mobiledatadownload.internal.FileGroupManager.GroupDownloadStatus; +import com.google.android.libraries.mobiledatadownload.internal.FileGroupsMetadata; +import com.google.android.libraries.mobiledatadownload.internal.MddTestUtil; +import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupUtil; +import com.google.common.util.concurrent.AsyncCallable; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.mobiledatadownload.LogEnumsProto.MddFileGroupDownloadStatus; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; +import com.google.mobiledatadownload.LogProto.MddFileGroupStatus; +import java.util.ArrayList; +import java.util.List; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.robolectric.RobolectricTestRunner; + +@RunWith(RobolectricTestRunner.class) +public class FileGroupStatsLoggerTest { + + private static final String TEST_GROUP = "test-group"; + private static final String TEST_GROUP_2 = "test-group-2"; + + private static final String TEST_PACKAGE = "test-package"; + + // This one has account + private static final GroupKey TEST_KEY = + GroupKey.newBuilder() + .setGroupName(TEST_GROUP) + .setOwnerPackage(TEST_PACKAGE) + .setAccount("some_account") + .build(); + + // This one does not have account + private static final GroupKey TEST_KEY_2 = + GroupKey.newBuilder().setGroupName(TEST_GROUP_2).setOwnerPackage(TEST_PACKAGE).build(); + + @Mock FileGroupManager mockFileGroupManager; + @Mock FileGroupsMetadata mockFileGroupsMetadata; + @Mock EventLogger mockEventLogger; + + private FileGroupStatsLogger fileGroupStatsLogger; + + @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + + @Captor + ArgumentCaptor<AsyncCallable<List<EventLogger.FileGroupStatusWithDetails>>> + fileGroupStatusAndDetailsListCaptor; + + @Before + public void setUp() throws Exception { + + fileGroupStatsLogger = + new FileGroupStatsLogger( + mockFileGroupManager, + mockFileGroupsMetadata, + mockEventLogger, + MoreExecutors.directExecutor()); + } + + @Test + public void fileGroupStatsLogging() throws Exception { + int daysSinceLastLog = 10; + + List<Pair<GroupKey, DataFileGroupInternal>> groups = new ArrayList<>(); + + // Add a downloaded group with version number 10. + DataFileGroupInternal fileGroupDownloaded = + MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 2).toBuilder() + .setFileGroupVersionNumber(10) + .setBuildId(10) + .setVariantId("test-variant") + .build(); + fileGroupDownloaded = + FileGroupUtil.setGroupNewFilesReceivedTimestamp(fileGroupDownloaded, 5000); + fileGroupDownloaded = FileGroupUtil.setDownloadedTimestampInMillis(fileGroupDownloaded, 10000); + + groups.add(Pair.create(TEST_KEY.toBuilder().setDownloaded(true).build(), fileGroupDownloaded)); + + // Add a pending download group for the same group name with version number 11. + DataFileGroupInternal fileGroupPending = + MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 3).toBuilder() + .setFileGroupVersionNumber(11) + .setStaleLifetimeSecs(0) + .setExpirationDateSecs(0) + .setBookkeeping(DataFileGroupBookkeeping.newBuilder().setStaleExpirationDate(0).build()) + .setBuildId(11) + .setVariantId("test-variant") + .build(); + fileGroupPending = FileGroupUtil.setGroupNewFilesReceivedTimestamp(fileGroupPending, 15000); + groups.add(Pair.create(TEST_KEY, fileGroupPending)); + when(mockFileGroupManager.getFileGroupDownloadStatus(fileGroupPending)) + .thenReturn(Futures.immediateFuture(GroupDownloadStatus.PENDING)); + + // Add a failed group to metadata with version 5. + DataFileGroupInternal fileGroupFailed = + MddTestUtil.createDataFileGroupInternal(TEST_GROUP_2, 3).toBuilder() + .setFileGroupVersionNumber(5) + .setStaleLifetimeSecs(0) + .setExpirationDateSecs(0) + .setBookkeeping(DataFileGroupBookkeeping.newBuilder().setStaleExpirationDate(0).build()) + .build(); + fileGroupFailed = FileGroupUtil.setGroupNewFilesReceivedTimestamp(fileGroupFailed, 12000); + groups.add(Pair.create(TEST_KEY_2, fileGroupFailed)); + when(mockFileGroupManager.getFileGroupDownloadStatus(fileGroupFailed)) + .thenReturn(Futures.immediateFuture(GroupDownloadStatus.FAILED)); + + when(mockFileGroupsMetadata.getAllFreshGroups()).thenReturn(Futures.immediateFuture(groups)); + + when(mockEventLogger.logMddFileGroupStats(any())).thenReturn(Futures.immediateVoidFuture()); + fileGroupStatsLogger.log(daysSinceLastLog).get(); + + verify(mockEventLogger, times(1)) + .logMddFileGroupStats(fileGroupStatusAndDetailsListCaptor.capture()); + + List<EventLogger.FileGroupStatusWithDetails> allFileGroupStatusAndDetailsList = + fileGroupStatusAndDetailsListCaptor.getValue().call().get(); + MddFileGroupStatus status1 = allFileGroupStatusAndDetailsList.get(0).fileGroupStatus(); + MddFileGroupStatus status2 = allFileGroupStatusAndDetailsList.get(1).fileGroupStatus(); + MddFileGroupStatus status3 = allFileGroupStatusAndDetailsList.get(2).fileGroupStatus(); + + DataDownloadFileGroupStats details1 = + allFileGroupStatusAndDetailsList.get(0).fileGroupDetails(); + DataDownloadFileGroupStats details2 = + allFileGroupStatusAndDetailsList.get(1).fileGroupDetails(); + DataDownloadFileGroupStats details3 = + allFileGroupStatusAndDetailsList.get(2).fileGroupDetails(); + + // Check that the downloaded group status is logged. + assertThat(details1.getFileGroupName()).isEqualTo(TEST_GROUP); + assertThat(details1.getOwnerPackage()).isEqualTo(TEST_PACKAGE); + assertThat(details1.getFileGroupVersionNumber()).isEqualTo(10); + assertThat(details1.getBuildId()).isEqualTo(10); + assertThat(details1.getVariantId()).isEqualTo("test-variant"); + assertThat(details1.getFileCount()).isEqualTo(2); + assertThat(details1.getInlineFileCount()).isEqualTo(0); + assertTrue(details1.getHasAccount()); + assertThat(status1.getFileGroupDownloadStatus()) + .isEqualTo(MddFileGroupDownloadStatus.Code.COMPLETE); + assertThat(status1.getGroupAddedTimestampInSeconds()).isEqualTo(5); + assertThat(status1.getGroupDownloadedTimestampInSeconds()).isEqualTo(10); + assertThat(status1.getDaysSinceLastLog()).isEqualTo(daysSinceLastLog); + + // Check that the pending group status is logged. + assertThat(details2.getFileGroupName()).isEqualTo(TEST_GROUP); + assertThat(details2.getFileGroupVersionNumber()).isEqualTo(11); + assertThat(details2.getBuildId()).isEqualTo(11); + assertThat(details2.getVariantId()).isEqualTo("test-variant"); + assertThat(details2.getOwnerPackage()).isEqualTo(TEST_PACKAGE); + assertThat(details2.getFileCount()).isEqualTo(3); + assertThat(details2.getInlineFileCount()).isEqualTo(0); + assertTrue(details2.getHasAccount()); + assertThat(status2.getFileGroupDownloadStatus()) + .isEqualTo(MddFileGroupDownloadStatus.Code.PENDING); + assertThat(status2.getGroupAddedTimestampInSeconds()).isEqualTo(15); + assertThat(status2.getGroupDownloadedTimestampInSeconds()).isEqualTo(-1); + assertThat(status2.getDaysSinceLastLog()).isEqualTo(daysSinceLastLog); + + // Check that the failed group status is logged. + assertThat(details3.getFileGroupName()).isEqualTo(TEST_GROUP_2); + assertThat(details3.getFileGroupVersionNumber()).isEqualTo(5); + assertThat(details3.getOwnerPackage()).isEqualTo(TEST_PACKAGE); + assertThat(details3.getFileCount()).isEqualTo(3); + assertThat(details3.getInlineFileCount()).isEqualTo(0); + assertFalse(details3.getHasAccount()); + assertThat(status3.getFileGroupDownloadStatus()) + .isEqualTo(MddFileGroupDownloadStatus.Code.FAILED); + assertThat(status3.getGroupAddedTimestampInSeconds()).isEqualTo(12); + assertThat(status3.getGroupDownloadedTimestampInSeconds()).isEqualTo(-1); + assertThat(status3.getDaysSinceLastLog()).isEqualTo(daysSinceLastLog); + } + + @Test + public void fileGroupStatsLogging_withInlineFiles() throws Exception { + int daysSinceLastLog = 10; + + List<Pair<GroupKey, DataFileGroupInternal>> groups = new ArrayList<>(); + + DataFile inlineFile1 = + DataFile.newBuilder() + .setFileId("inline-file") + .setUrlToDownload("inlinefile:sha1:checksum") + .setChecksum("checksum") + .setByteSize(10) + .build(); + DataFile inlineFile2 = + DataFile.newBuilder() + .setFileId("inline-file-2") + .setUrlToDownload("inlinefile:sha1:checksum2") + .setChecksum("checksum2") + .setByteSize(11) + .build(); + + // Add a downloaded group with version number 10 and inline file + DataFileGroupInternal fileGroupDownloaded = + MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 2).toBuilder() + .setFileGroupVersionNumber(10) + .setBuildId(10) + .setVariantId("test-variant") + .addFile(inlineFile1) + .addFile(inlineFile2) + .build(); + fileGroupDownloaded = + FileGroupUtil.setGroupNewFilesReceivedTimestamp(fileGroupDownloaded, 5000); + fileGroupDownloaded = FileGroupUtil.setDownloadedTimestampInMillis(fileGroupDownloaded, 10000); + + groups.add(Pair.create(TEST_KEY.toBuilder().setDownloaded(true).build(), fileGroupDownloaded)); + + // Add a pending download group for the same group name with version number 11 and inline file. + DataFileGroupInternal fileGroupPending = + MddTestUtil.createDataFileGroupInternal(TEST_GROUP, 3).toBuilder() + .setFileGroupVersionNumber(11) + .setStaleLifetimeSecs(0) + .setExpirationDateSecs(0) + .setBookkeeping(DataFileGroupBookkeeping.newBuilder().setStaleExpirationDate(0).build()) + .setBuildId(11) + .setVariantId("test-variant") + .addFile(inlineFile1) + .build(); + fileGroupPending = FileGroupUtil.setGroupNewFilesReceivedTimestamp(fileGroupPending, 15000); + groups.add(Pair.create(TEST_KEY, fileGroupPending)); + when(mockFileGroupManager.getFileGroupDownloadStatus(fileGroupPending)) + .thenReturn(Futures.immediateFuture(GroupDownloadStatus.PENDING)); + + // Add a failed group to metadata with version 5 with no inline files. + DataFileGroupInternal fileGroupFailed = + MddTestUtil.createDataFileGroupInternal(TEST_GROUP_2, 3).toBuilder() + .setFileGroupVersionNumber(5) + .setStaleLifetimeSecs(0) + .setExpirationDateSecs(0) + .setBookkeeping(DataFileGroupBookkeeping.newBuilder().setStaleExpirationDate(0).build()) + .build(); + fileGroupFailed = FileGroupUtil.setGroupNewFilesReceivedTimestamp(fileGroupFailed, 12000); + groups.add(Pair.create(TEST_KEY_2, fileGroupFailed)); + when(mockFileGroupManager.getFileGroupDownloadStatus(fileGroupFailed)) + .thenReturn(Futures.immediateFuture(GroupDownloadStatus.FAILED)); + + when(mockFileGroupsMetadata.getAllFreshGroups()).thenReturn(Futures.immediateFuture(groups)); + when(mockEventLogger.logMddFileGroupStats(any())).thenReturn(Futures.immediateVoidFuture()); + + fileGroupStatsLogger.log(daysSinceLastLog).get(); + + verify(mockEventLogger, times(1)) + .logMddFileGroupStats(fileGroupStatusAndDetailsListCaptor.capture()); + + List<EventLogger.FileGroupStatusWithDetails> allFileGroupStatusAndDetailsList = + fileGroupStatusAndDetailsListCaptor.getValue().call().get(); + MddFileGroupStatus status1 = allFileGroupStatusAndDetailsList.get(0).fileGroupStatus(); + MddFileGroupStatus status2 = allFileGroupStatusAndDetailsList.get(1).fileGroupStatus(); + MddFileGroupStatus status3 = allFileGroupStatusAndDetailsList.get(2).fileGroupStatus(); + + DataDownloadFileGroupStats details1 = + allFileGroupStatusAndDetailsList.get(0).fileGroupDetails(); + DataDownloadFileGroupStats details2 = + allFileGroupStatusAndDetailsList.get(1).fileGroupDetails(); + DataDownloadFileGroupStats details3 = + allFileGroupStatusAndDetailsList.get(2).fileGroupDetails(); + + // Check that the downloaded group status is logged. + assertThat(details1.getFileGroupName()).isEqualTo(TEST_GROUP); + assertThat(details1.getOwnerPackage()).isEqualTo(TEST_PACKAGE); + assertThat(details1.getFileGroupVersionNumber()).isEqualTo(10); + assertThat(details1.getBuildId()).isEqualTo(10); + assertThat(details1.getVariantId()).isEqualTo("test-variant"); + assertThat(details1.getFileCount()).isEqualTo(4); + assertThat(details1.getInlineFileCount()).isEqualTo(2); + assertTrue(details1.getHasAccount()); + assertThat(status1.getFileGroupDownloadStatus()) + .isEqualTo(MddFileGroupDownloadStatus.Code.COMPLETE); + assertThat(status1.getGroupAddedTimestampInSeconds()).isEqualTo(5); + assertThat(status1.getGroupDownloadedTimestampInSeconds()).isEqualTo(10); + assertThat(status1.getDaysSinceLastLog()).isEqualTo(daysSinceLastLog); + + // Check that the pending group status is logged. + assertThat(details2.getFileGroupName()).isEqualTo(TEST_GROUP); + assertThat(details2.getFileGroupVersionNumber()).isEqualTo(11); + assertThat(details2.getBuildId()).isEqualTo(11); + assertThat(details2.getVariantId()).isEqualTo("test-variant"); + assertThat(details2.getOwnerPackage()).isEqualTo(TEST_PACKAGE); + assertThat(details2.getFileCount()).isEqualTo(4); + assertThat(details2.getInlineFileCount()).isEqualTo(1); + assertTrue(details2.getHasAccount()); + assertThat(status2.getFileGroupDownloadStatus()) + .isEqualTo(MddFileGroupDownloadStatus.Code.PENDING); + assertThat(status2.getGroupAddedTimestampInSeconds()).isEqualTo(15); + assertThat(status2.getGroupDownloadedTimestampInSeconds()).isEqualTo(-1); + assertThat(status2.getDaysSinceLastLog()).isEqualTo(daysSinceLastLog); + + // Check that the failed group status is logged. + assertThat(details3.getFileGroupName()).isEqualTo(TEST_GROUP_2); + assertThat(details3.getFileGroupVersionNumber()).isEqualTo(5); + assertThat(details3.getOwnerPackage()).isEqualTo(TEST_PACKAGE); + assertThat(details3.getFileCount()).isEqualTo(3); + assertThat(details3.getInlineFileCount()).isEqualTo(0); + assertFalse(details3.getHasAccount()); + assertThat(status3.getFileGroupDownloadStatus()) + .isEqualTo(MddFileGroupDownloadStatus.Code.FAILED); + assertThat(status3.getGroupAddedTimestampInSeconds()).isEqualTo(12); + assertThat(status3.getGroupDownloadedTimestampInSeconds()).isEqualTo(-1); + assertThat(status3.getDaysSinceLastLog()).isEqualTo(daysSinceLastLog); + } +}
\ No newline at end of file diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/LogSamplerTest.java b/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/LogSamplerTest.java new file mode 100644 index 0000000..69c8186 --- /dev/null +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/LogSamplerTest.java @@ -0,0 +1,233 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.libraries.mobiledatadownload.internal.logging; + +import static com.google.android.libraries.mobiledatadownload.internal.logging.SharedPreferencesLoggingState.SALT_KEY; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assume.assumeTrue; + +import android.content.Context; +import android.content.SharedPreferences; +import androidx.test.core.app.ApplicationProvider; +import com.google.android.libraries.mobiledatadownload.Flags; +import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; +import com.google.android.libraries.mobiledatadownload.file.backends.JavaFileBackend; +import com.google.android.libraries.mobiledatadownload.testing.FakeTimeSource; +import com.google.common.base.Optional; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.mobiledatadownload.LogProto.StableSamplingInfo; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.concurrent.Executors; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.ParameterizedRobolectricTestRunner; +import org.robolectric.ParameterizedRobolectricTestRunner.Parameter; +import org.robolectric.ParameterizedRobolectricTestRunner.Parameters; + +@RunWith(ParameterizedRobolectricTestRunner.class) +public final class LogSamplerTest { + @Parameter(value = 0) + public boolean stableLoggingEnabled; + + @Parameters(name = "stableLoggingEnabled = {0}") + public static List<Boolean> parameters() { + return Arrays.asList(true, false); + } + + private LoggingStateStore loggingStateStore; + private SharedPreferences loggingStateSharedPrefs; + private static final ListeningExecutorService executorService = + MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); + private LogSampler logSampler; + + private static final FakeTimeSource timeSource = new FakeTimeSource(); + private Context context; + + // Seed for first long + private static final int LOGS_AT_1_PERCENT_SEED = 750; // -5772485602628857500 + + private static final int ONE_PERCENT_SAMPLE_INTERVAL = 100; + private static final int TEN_PERCENT_SAMPLE_INTERVAL = 10; + private static final int ONE_HUNDRED_PERCENT_SAMPLE_INTERVAL = 1; + private static final int NEVER_SAMPLE_INTERVAL = 0; + + @Before + public void setUp() throws Exception { + context = ApplicationProvider.getApplicationContext(); + + SynchronousFileStorage fileStorage = + new SynchronousFileStorage(Arrays.asList(new JavaFileBackend())); + + loggingStateSharedPrefs = context.getSharedPreferences("loggingStateSharedPrefs", 0); + loggingStateStore = + SharedPreferencesLoggingState.create( + () -> loggingStateSharedPrefs, timeSource, executorService, new Random()); + + logSampler = constructLogSampler(0); + } + + @Test + public void shouldLog_withInvalidSamplingRate_returnsAbsent() throws Exception { + int invalidSamplingRate = -1; + Optional<StableSamplingInfo> samplingInfo = + logSampler.shouldLog(invalidSamplingRate, Optional.of(loggingStateStore)).get(); + + assertThat(samplingInfo).isAbsent(); + } + + @Test + public void shouldLog_with0SamplingRate_returnsAbsent() throws Exception { + Optional<StableSamplingInfo> samplingInfo = + logSampler.shouldLog(NEVER_SAMPLE_INTERVAL, Optional.of(loggingStateStore)).get(); + + assertThat(samplingInfo).isAbsent(); + } + + @Test + public void shouldLog_stable_with1PercentGroup_logsAt1Percent() throws Exception { + assumeTrue(stableLoggingEnabled); + setStableSamplingRandomNumber(100); // 100 % 100 = 0 + + Optional<StableSamplingInfo> samplingInfo = + logSampler.shouldLog(ONE_PERCENT_SAMPLE_INTERVAL, Optional.of(loggingStateStore)).get(); + + assertThat(samplingInfo).isPresent(); + assertThat(samplingInfo.get().getStableSamplingUsed()).isTrue(); + assertThat(samplingInfo.get().getPartOfAlwaysLoggingGroup()).isTrue(); + assertThat(samplingInfo.get().getInvalidSamplingRateUsed()).isFalse(); + } + + @Test + public void shouldLog_stable_with1PercentGroup_logsAt10Percent() throws Exception { + assumeTrue(stableLoggingEnabled); + setStableSamplingRandomNumber(100); // 100 % 100 = 0 + + Optional<StableSamplingInfo> samplingInfo = + logSampler.shouldLog(TEN_PERCENT_SAMPLE_INTERVAL, Optional.of(loggingStateStore)).get(); + + assertThat(samplingInfo).isPresent(); + assertThat(samplingInfo.get().getStableSamplingUsed()).isTrue(); + assertThat(samplingInfo.get().getPartOfAlwaysLoggingGroup()).isTrue(); + assertThat(samplingInfo.get().getInvalidSamplingRateUsed()).isFalse(); + } + + @Test + public void shouldLog_stable_with10PercentGroup_doesntLogAt1Percent() throws Exception { + assumeTrue(stableLoggingEnabled); + setStableSamplingRandomNumber(10); // 10 % 100 = 10 + + Optional<StableSamplingInfo> samplingInfo = + logSampler.shouldLog(ONE_PERCENT_SAMPLE_INTERVAL, Optional.of(loggingStateStore)).get(); + + assertThat(samplingInfo).isAbsent(); + } + + @Test + public void shouldLog_stable_with10PercentGroup_logsAt10Percent() throws Exception { + assumeTrue(stableLoggingEnabled); + setStableSamplingRandomNumber(10); // 10 % 100 = 10 + + Optional<StableSamplingInfo> samplingInfo = + logSampler.shouldLog(TEN_PERCENT_SAMPLE_INTERVAL, Optional.of(loggingStateStore)).get(); + + assertThat(samplingInfo).isPresent(); + assertThat(samplingInfo.get().getStableSamplingUsed()).isTrue(); + assertThat(samplingInfo.get().getPartOfAlwaysLoggingGroup()).isFalse(); + assertThat(samplingInfo.get().getInvalidSamplingRateUsed()).isFalse(); + } + + @Test + public void shouldLog_stable_withIncompatibleSamplingRate_isMarkedAsIncompatible() + throws Exception { + assumeTrue(stableLoggingEnabled); + setStableSamplingRandomNumber(77); + + Optional<StableSamplingInfo> samplingInfo = + logSampler.shouldLog(77, Optional.of(loggingStateStore)).get(); + + assertThat(samplingInfo).isPresent(); + assertThat(samplingInfo.get().getStableSamplingUsed()).isTrue(); + assertThat(samplingInfo.get().getInvalidSamplingRateUsed()).isTrue(); + assertThat(samplingInfo.get().getPartOfAlwaysLoggingGroup()).isFalse(); + } + + @Test + public void shouldLog_with100Percent_logsAt100Percent() throws Exception { + Optional<StableSamplingInfo> samplingInfo1 = + logSampler + .shouldLog(ONE_HUNDRED_PERCENT_SAMPLE_INTERVAL, Optional.of(loggingStateStore)) + .get(); + Optional<StableSamplingInfo> samplingInfo2 = + logSampler + .shouldLog(ONE_HUNDRED_PERCENT_SAMPLE_INTERVAL, Optional.of(loggingStateStore)) + .get(); + + assertThat(samplingInfo1).isPresent(); + assertThat(samplingInfo2).isPresent(); + assertThat(samplingInfo1.get().getStableSamplingUsed()).isEqualTo(stableLoggingEnabled); + assertThat(samplingInfo2.get().getStableSamplingUsed()).isEqualTo(stableLoggingEnabled); + } + + @Test + public void shouldLog_event_changesPerEvent() throws Exception { + assumeTrue(!stableLoggingEnabled); + + LogSampler logSampler = constructLogSampler(LOGS_AT_1_PERCENT_SEED); + checkState( + logSampler + .shouldLog(ONE_PERCENT_SAMPLE_INTERVAL, Optional.of(loggingStateStore)) + .get() + .isPresent()); + + assertThat( + logSampler.shouldLog(ONE_PERCENT_SAMPLE_INTERVAL, Optional.of(loggingStateStore)).get()) + .isAbsent(); + } + + @Test + public void shouldLog_stable_withoutLoggingStateStore_usesPerEvent() throws Exception { + assumeTrue(stableLoggingEnabled); + + Optional<StableSamplingInfo> stableSamplingInfo = + logSampler.shouldLog(ONE_HUNDRED_PERCENT_SAMPLE_INTERVAL, Optional.absent()).get(); + + assertThat(stableSamplingInfo).isPresent(); + assertThat(stableSamplingInfo.get().getStableSamplingUsed()).isFalse(); + } + + private LogSampler constructLogSampler(int seed) { + return new LogSampler( + new Flags() { + @Override + public boolean enableRngBasedDeviceStableSampling() { + return stableLoggingEnabled; + } + }, + new Random(seed)); + } + + private void setStableSamplingRandomNumber(int randomNumber) throws Exception { + SharedPreferences.Editor editor = loggingStateSharedPrefs.edit(); + editor.putLong(SALT_KEY, randomNumber); + assumeTrue(editor.commit()); + } +}
\ No newline at end of file diff --git a/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/MddEventLoggerTest.java b/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/MddEventLoggerTest.java new file mode 100644 index 0000000..d0bbb5c --- /dev/null +++ b/javatests/com/google/android/libraries/mobiledatadownload/internal/logging/MddEventLoggerTest.java @@ -0,0 +1,186 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.libraries.mobiledatadownload.internal.logging; + +import static com.google.common.util.concurrent.Futures.immediateFuture; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +import android.content.Context; +import android.content.SharedPreferences; +import androidx.test.core.app.ApplicationProvider; +import com.google.android.libraries.mobiledatadownload.Logger; +import com.google.android.libraries.mobiledatadownload.internal.logging.EventLogger.FileGroupStatusWithDetails; +import com.google.android.libraries.mobiledatadownload.testing.FakeTimeSource; +import com.google.android.libraries.mobiledatadownload.testing.TestFlags; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; +import com.google.mobiledatadownload.LogEnumsProto.MddFileGroupDownloadStatus; +import com.google.mobiledatadownload.LogProto.AndroidClientInfo; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; +import com.google.mobiledatadownload.LogProto.MddDeviceInfo; +import com.google.mobiledatadownload.LogProto.MddFileGroupStatus; +import com.google.mobiledatadownload.LogProto.MddLogData; +import com.google.mobiledatadownload.LogProto.StableSamplingInfo; +import java.security.SecureRandom; +import java.util.Random; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.robolectric.RobolectricTestRunner; + +@RunWith(RobolectricTestRunner.class) +public class MddEventLoggerTest { + + @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + + private static final int SOME_MODULE_VERSION = 42; + private static final int SAMPLING_ALWAYS = 1; + private static final int SAMPLING_NEVER = 0; + + @Mock private Logger mockLogger; + private MddEventLogger mddEventLogger; + + private final Context context = ApplicationProvider.getApplicationContext(); + private final TestFlags flags = new TestFlags(); + + @Before + public void setUp() throws Exception { + SharedPreferences loggingStateSharedPrefs = + context.getSharedPreferences("loggingStateSharedPrefs", 0); + mddEventLogger = + new MddEventLogger( + context, + mockLogger, + SOME_MODULE_VERSION, + new LogSampler(flags, new SecureRandom()), + flags); + mddEventLogger.setLoggingStateStore( + SharedPreferencesLoggingState.create( + () -> loggingStateSharedPrefs, new FakeTimeSource(), directExecutor(), new Random(0))); + } + + private MddLogData.Builder newLogDataBuilderWithClientInfo() { + return MddLogData.newBuilder() + .setAndroidClientInfo( + AndroidClientInfo.newBuilder() + .setModuleVersion(SOME_MODULE_VERSION) + .setHostPackageName(context.getPackageName())); + } + + @Test + public void testSampleInterval_zero_none() { + assertFalse(LogUtil.shouldSampleInterval(0)); + } + + @Test + public void testSampleInterval_negative_none() { + assertFalse(LogUtil.shouldSampleInterval(-1)); + } + + @Test + public void testSampleInterval_always() { + assertTrue(LogUtil.shouldSampleInterval(1)); + } + + @Test + public void testLogMddEvents_noLog() throws Exception { + overrideDefaultSampleInterval(SAMPLING_NEVER); + + DataDownloadFileGroupStats fileGroupStats = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName("fileGroup") + .setFileGroupVersionNumber(1) + .setBuildId(123) + .setVariantId("testVariant") + .build(); + MddFileGroupStatus fileGroupStatus = + MddFileGroupStatus.newBuilder() + .setFileGroupDownloadStatus(MddFileGroupDownloadStatus.Code.COMPLETE) + .build(); + FileGroupStatusWithDetails fileGroupStatusWithDetails = + FileGroupStatusWithDetails.create(fileGroupStatus, fileGroupStats); + + mddEventLogger + .logMddFileGroupStats(() -> immediateFuture(ImmutableList.of(fileGroupStatusWithDetails))) + .get(); + + verifyNoInteractions(mockLogger); + } + + @Test + public void testLogMddEvents() throws Exception { + overrideDefaultSampleInterval(SAMPLING_ALWAYS); + + DataDownloadFileGroupStats fileGroupStats = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName("fileGroup") + .setFileGroupVersionNumber(1) + .setBuildId(123) + .setVariantId("testVariant") + .build(); + MddFileGroupStatus fileGroupStatus = + MddFileGroupStatus.newBuilder() + .setFileGroupDownloadStatus(MddFileGroupDownloadStatus.Code.COMPLETE) + .build(); + FileGroupStatusWithDetails fileGroupStatusWithDetails = + FileGroupStatusWithDetails.create(fileGroupStatus, fileGroupStats); + + MddLogData expectedData = + newLogDataBuilderWithClientInfo() + .setSamplingInterval(SAMPLING_ALWAYS) + .setDataDownloadFileGroupStats(fileGroupStats) + .setMddFileGroupStatus(fileGroupStatus) + .setDeviceInfo(MddDeviceInfo.newBuilder().setDeviceStorageLow(false)) + .setStableSamplingInfo(getStableSamplingInfo()) + .build(); + + mddEventLogger + .logMddFileGroupStats(() -> immediateFuture(ImmutableList.of(fileGroupStatusWithDetails))) + .get(); + + verify(mockLogger) + .log(eq(expectedData), eq(MddClientEvent.Code.DATA_DOWNLOAD_FILE_GROUP_STATUS_VALUE)); + } + + private void overrideDefaultSampleInterval(int sampleInterval) { + flags.mddDefaultSampleInterval = Optional.of(sampleInterval); + flags.groupStatsLoggingSampleInterval = Optional.of(sampleInterval); + } + + private StableSamplingInfo getStableSamplingInfo() { + if (flags.enableRngBasedDeviceStableSampling()) { + return StableSamplingInfo.newBuilder() + .setStableSamplingUsed(true) + .setStableSamplingFirstEnabledTimestampMs(0) + .setPartOfAlwaysLoggingGroup(false) + .setInvalidSamplingRateUsed(false) + .build(); + } + + return StableSamplingInfo.newBuilder().setStableSamplingUsed(false).build(); + } +}
\ No newline at end of file diff --git a/mobile-data-download.iml b/mobile-data-download.iml deleted file mode 100644 index afa4d1e..0000000 --- a/mobile-data-download.iml +++ /dev/null @@ -1,13 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<module type="JAVA_MODULE" version="4"> - <component name="NewModuleRootManager" inherit-compiler-output="true"> - <exclude-output /> - <content url="file://$MODULE_DIR$"> - <sourceFolder url="file://$MODULE_DIR$/android-annotation-stubs" isTestSource="false" packagePrefix="__PACKAGE__" /> - <sourceFolder url="file://$MODULE_DIR$/java" isTestSource="false" /> - <sourceFolder url="file://$MODULE_DIR$/javatests" isTestSource="false" /> - </content> - <orderEntry type="inheritedJdk" /> - <orderEntry type="sourceFolder" forTests="false" /> - </component> -</module>
\ No newline at end of file diff --git a/proto/Android.bp b/proto/Android.bp index 6c3e681..198768f 100644 --- a/proto/Android.bp +++ b/proto/Android.bp @@ -41,6 +41,6 @@ java_library { apex_available: [ "//apex_available:platform", "com.android.adservices", - "com.android.ondevicepersonalization", + "com.android.ondevicepersonalization", ], } diff --git a/proto/atoms.proto b/proto/atoms.proto new file mode 100644 index 0000000..293b466 --- /dev/null +++ b/proto/atoms.proto @@ -0,0 +1,56 @@ +syntax = "proto2"; + +package mobiledatadownload.logs; + +option java_multiple_files = true; +option java_package = "com.google.mobiledatadownload"; +option java_outer_classname = "AtomsProto"; + +/** + * These protos are duplicates of the MobileDataDownload protos logged as + * MODE_BYTES in go/atoms.proto. + * TODO(b/243579271): remove this duplication + */ +/** Shared data among MobileDataDownload statistics. Not meant to be a top level + * atom proto.*/ +message MobileDataDownloadFileGroupStats { + // The name of the file group. This a string set server side used to retrieve + // the files. Does not contain PII. + optional string file_group_name = 1; + // Allows the clients to identify a file group based on a given set of + // properties. This string is set server side and does not contain PII. + optional string variant_id = 2; + // Identifier for the data file group created to identify the version of the + // file group. + optional int64 build_id = 3; + // The number of files in the file group. + optional int32 file_count = 4; + // Whether the file group has an account associated with it. + optional bool has_account = 5; + // Inverse of the sampling rate used to sample this event. + optional int32 sampling_interval = 6; + // Note: we do not have owner_package since that's already transmitted. +} + +/** + * Container for MobileDataDownloadFileGroupStorageStats + */ +message MobileDataDownloadStorageStats { + repeated MobileDataDownloadFileGroupStorageStats + mobile_data_download_file_group_storage_stats = 1; +} + +/** + * Storage stats for a single file group. This is logged as a nested field and + * is not meant to be logged as a top level proto. + */ +message MobileDataDownloadFileGroupStorageStats { + // The total number of bytes used by this file group + optional int64 total_bytes_used = 1; + // The total number of inline file bytes used by this file group + optional int64 total_inline_bytes_used = 2; + // The number of bytes used for the downloaded version of the file group + optional int64 downloaded_group_bytes_used = 3; + // Specifies which file group this storage is associated with + optional MobileDataDownloadFileGroupStats file_group_stats = 4; +} diff --git a/proto/log_enums.proto b/proto/log_enums.proto new file mode 100644 index 0000000..a6d4d63 --- /dev/null +++ b/proto/log_enums.proto @@ -0,0 +1,50 @@ +syntax = "proto3"; + +package mobiledatadownload.logs; + +option java_package = "com.google.mobiledatadownload"; +option java_outer_classname = "LogEnumsProto"; + +// MDD client side events used for logging with MddLogData. +// +// Each feature gets a range of 1000 enums starting at X000. 1st enum specifies +// if the feature is enabled. Subsequent 999 enums can be used to define events +// within the feature. Unused enums in the range are left for future use for +// the *same* feature. +// If a feature ever exhausts it's quota of enums, it should be migrated to a +// new range of contiguous 2000 enums by deprecating the existing enums. +// +// Enums should never be deleted or reused, but they can be renamed*. Old enums +// should be left in their position with [deprecated=true] attribute. +// +// * For renaming enums, see <internal> +message MddClientEvent { + enum Code { + // Do not use this default value. + EVENT_CODE_UNSPECIFIED = 0; + + // Events for Mobile Data Download (<internal>) (1000-1999). + // Next enum for data download: 1114 + + // Log in a periodic tasks. + // Logged with DataDownloadFileGroupStats, MddFileGroupStatus. + DATA_DOWNLOAD_FILE_GROUP_STATUS = 1044; + + reserved 1000 to 1043, 1045 to 1113; + + reserved 2000 to 2999, 3000 to 3999, 4000 to 4099, 4100 to 4199, + 5000 to 5999, 6000 to 6999, 7000 to 7999, 8000 to 8999, 9000 to 9999, + 10000 to 10999, 11000 to 11999, 12000 to 12999, 13000, 13999, + 14000 to 14999, 15000 to 15999, 16000 to 16999, 17000 to 17999, + 18000 to 18999, 19000 to 19999; + } +} + +message MddFileGroupDownloadStatus { + enum Code { + INVALID = 0; + COMPLETE = 1; + PENDING = 2; + FAILED = 3; + } +}
\ No newline at end of file diff --git a/proto/logs.proto b/proto/logs.proto new file mode 100644 index 0000000..c4c2f6e --- /dev/null +++ b/proto/logs.proto @@ -0,0 +1,203 @@ +// Logging protos for MobileDataDownload + +syntax = "proto2"; + +package mobiledatadownload.logs; + +import "log_enums.proto"; + +// option jspb_use_correct_proto2_semantics = false; // <internal> +option java_package = "com.google.mobiledatadownload"; +option java_outer_classname = "LogProto"; + +// Info about the Android client that logged. +// Next tag: 3 +message AndroidClientInfo { + // Version of the module we are currently running. aMDD will log its own + // version that it shares between GMSCore module and library. + + optional int32 module_version = 1 [default = -1]; + + // Package name of the hosting application. + // It is to differentiate logs from GMS service and library. + optional string host_package_name = 2; +} + +// Attributes of the device and/or MDD +// Recommended to log this message with each MDD log defined below. This will +// allow slicing MDD stats on the state of the device. +// +// TODO: Make Fields of this proto available as RASTA conditions for +// experimentation. +// +// Next tag: 3 +message MddDeviceInfo { + // Indicates low storage space condition on the device. + // Currently in O-, it is the result of Android's ACTION_DEVICE_STORAGE_LOW + // intent when the storage state was logged. + // For O+, MDD will define its own threshold for low storage: b/77856395 + optional bool device_storage_low = 1; + + reserved 2; +} + +// Metadata associated with each data download event specific to a file group. +// +// Next tag: 9 +message DataDownloadFileGroupStats { + // Name of the file group. + optional string file_group_name = 1; + + // Client set version number used to identify the file group. + // Note that this does not uniquely identify the contents of the file group. + // It simply reflects a snapshot of client config changes. + // For example: say there's a file group 'language-detector-model' that + // downloads a different file per user locale. + // data_file_group { + // file_group_name = 'language-detector-model' + // file_group_version_number = 1 + // file { + // url = 'en-model' + // } + // } + // data_file_group { + // file_group_name = 'language-detector-model' + // file_group_version_number = 1 + // file { + // url = 'es-model' + // } + // } + // Note that even though the actual contents of the file group are different + // for each locale, the version is the same because this config was pushed + // at the same snapshot. + optional int32 file_group_version_number = 2; + + // The package name of the group owner. + optional string owner_package = 3; + + // The total number of files in the file group. + // + // NOTE: This count is only included for storage and file group stats logging + optional int32 file_count = 4; + + // The number of inline files in the file group. + // + // NOTE: This count is only included for storage and file group stats logging + optional int32 inline_file_count = 8; + + // Whether the file group has an account associated with it or not. This will + // allow us to slice metrics by having account or not. For more info see + // cs/symbol:mdi.download.internal.GroupKey.account + optional bool has_account = 5; + + // The build id for the file group. Unique identifier for a file group config + // created when using MDD Ingress API. + // For more details see <internal>. + optional int64 build_id = 6; + + // The VariantID of the DataFileGroup. This is set up server side via code + // review. For more details see <internal>. + // Examples: "en", "de-universal", etc. + optional string variant_id = 7; +} + +// The status of a MDD file group. This data is logged periodically to get +// a snapshot of the status of a file group on devices. +// Next tag: 5 +message MddFileGroupStatus { + // Download status of the whole file group. This is an AND over the + // download status of each file within the file group. + optional MddFileGroupDownloadStatus.Code file_group_download_status = 1; + + // Time since epoch when this file group was first added to mdd. + // + // Set to -1 if this time is unknown (for legacy groups). + // + // This matches the field "group_new_files_received_timestamp" in metadata + // <internal> + optional int64 group_added_timestamp_in_seconds = 2; + + // Time since epoch when this file group was downloaded by mdd. + // + // Set to -1 if this time is unknown (for legacy groups) and non-downloaded + // groups + // + // This matches the field "group_downloaded_timestamp_in_millis" in metadata + // <internal> + optional int64 group_downloaded_timestamp_in_seconds = 3; + + // Number of days since this status was last logged (number of days since + // daily maintenance was last run). + // + // Set to -1 if there is an invalid or unknown value. + // + // See <internal> for more info. + optional int32 days_since_last_log = 4; +} + +// Various health reports. +// Ideally, this should be defined as an empty message that allows extensions +// and each inner proto should be defined as its extension. +// TODO: Figure out if there are limitations in nano-proto that might +// not allow this. +// +// Next tag: 74 +message MddLogData { + // MDD data download file group stats. + optional DataDownloadFileGroupStats data_download_file_group_stats = 10; + + // Sampling interval used while logging this message. The default value 0 is + // not a valid value for messages using this filed since it a special value + // denoting that message should not be logged. Hence value of 0 means it was + // not filled in. + optional int64 sampling_interval = 21; + + // Additional info necessary for stable sampling. + optional StableSamplingInfo stable_sampling_info = 72; + + // Data download file group download status (logged periodically). + optional MddFileGroupStatus mdd_file_group_status = 32; + + // Attributes of the device and/or MDD at the time we log other stats. + optional MddDeviceInfo device_info = 40; + + // Android client info. + optional AndroidClientInfo android_client_info = 51; + + reserved 1 to 9, 11 to 20, 22 to 31, 33 to 39, 41 to 50, 52 to 71, 73; +} + +// Info on sampling method used for log events. Stable sampling means if a +// device is selected to log, it will log all events. See <internal> +// Next tag: 5 +message StableSamplingInfo { + // Whether a stable sampling method (as described in <internal>) + // was used. + optional bool stable_sampling_used = 1; + + // When stable sampling was first enabled on the device. This will be useful + // when rolling out and processing logs over multiple days. + optional int64 stable_sampling_first_enabled_timestamp_ms = 2; + + // Whether or not this device would log with the 1% cohort. Devices in the 1% + // cohort are *always* logging, and will always log without further code + // changes. When a device has this set to true, it's expected that the device + // is *always* logging and the sampling rate should not be changed to + // something that results in this device being excluded from the logging group + // (see invalid_sampling_rate_used). + // + // Most dashboards/metrics depending on linking together multiple events from + // the same device should filter to devices/events that have this set to true + // (with the caveat that we won't use all data from all devices reporting). + // This is useful when we need to change sampling rates, e.g. for an + // experiment. + optional bool part_of_always_logging_group = 3; + + // If we are using stable sampling, we expect a sampling rate where '100 % + // sample_interval == 0'. This ensures that devices logging at 1 percent + // sampling interval, will continue to log at other chosen sampling rates too. + // This should only be false if we've incorrectly configured our sampling + // rates (e.g. a sampling rate of 101 would mean that the 1 percent cohort + // devices would not log). + optional bool invalid_sampling_rate_used = 4; +}
\ No newline at end of file |