test(storage): add methods to retry conformance tests (#4997)

2 files changed, 469 insertions, 0 deletions

diff --git a/storage/internal/test/conformance/retry_tests.json b/storage/internal/test/conformance/retry_tests.json
index 0fe114775..09f384c5d 100644
--- a/storage/internal/test/conformance/retry_tests.json
+++ b/storage/internal/test/conformance/retry_tests.json
@@ -40,6 +40,156 @@
       ],
       "preconditionProvided": false,
       "expectSuccess": true
+    },
+    {
+      "id": 2,
+      "description": "conditionally_idempotent_retries_when_precondition_is_present",
+      "cases": [
+        {
+          "instructions": ["return-503", "return-503"]
+        },
+        {
+          "instructions": ["return-reset-connection", "return-reset-connection"]
+        },
+        {
+          "instructions": ["return-reset-connection", "return-503"]
+        }
+      ],
+      "methods": [
+        {"name": "storage.buckets.patch",         "resources": ["BUCKET"]},
+        {"name": "storage.buckets.setIamPolicy",  "resources": ["BUCKET"]},
+        {"name": "storage.buckets.update",        "resources": ["BUCKET"]},
+        {"name": "storage.hmacKey.update",        "resources": ["HMAC_KEY"]},
+        {"name": "storage.objects.compose",       "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.objects.copy",          "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.objects.delete",        "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.objects.insert",        "resources": ["BUCKET"]},
+        {"name": "storage.objects.patch",         "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.objects.rewrite",       "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.objects.update",        "resources": ["BUCKET", "OBJECT"]}
+      ],
+      "preconditionProvided": true,
+      "expectSuccess": true
+    },
+    {
+      "id": 3,
+      "description": "conditionally_idempotent_no_retries_when_precondition_is_absent",
+      "cases": [
+        {
+          "instructions": ["return-503"]
+        },
+        {
+          "instructions": ["return-reset-connection"]
+        }
+      ],
+      "methods": [
+        {"name": "storage.buckets.patch",         "resources": ["BUCKET"]},
+        {"name": "storage.buckets.setIamPolicy",  "resources": ["BUCKET"]},
+        {"name": "storage.buckets.update",        "resources": ["BUCKET"]},
+        {"name": "storage.hmacKey.update",        "resources": ["HMAC_KEY"]},
+        {"name": "storage.objects.compose",       "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.objects.copy",          "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.objects.delete",        "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.objects.insert",        "resources": ["BUCKET"]},
+        {"name": "storage.objects.patch",         "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.objects.rewrite",       "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.objects.update",        "resources": ["BUCKET", "OBJECT"]}
+      ],
+      "preconditionProvided": false,
+      "expectSuccess": false
+    },
+    {
+      "id": 4,
+      "description": "non_idempotent",
+      "cases": [
+        {
+          "instructions": ["return-503"]
+        },
+        {
+          "instructions": ["return-reset-connection"]
+        }
+      ],
+      "methods": [
+        {"name": "storage.bucket_acl.delete",             "resources": ["BUCKET"]},
+        {"name": "storage.bucket_acl.insert",             "resources": ["BUCKET"]},
+        {"name": "storage.bucket_acl.patch",              "resources": ["BUCKET"]},
+        {"name": "storage.bucket_acl.update",             "resources": ["BUCKET"]},
+        {"name": "storage.default_object_acl.delete",     "resources": ["BUCKET"]},
+        {"name": "storage.default_object_acl.insert",     "resources": ["BUCKET"]},
+        {"name": "storage.default_object_acl.patch",      "resources": ["BUCKET"]},
+        {"name": "storage.default_object_acl.update",     "resources": ["BUCKET"]},
+        {"name": "storage.hmacKey.create",                "resources": []},
+        {"name": "storage.notifications.insert",          "resources": ["BUCKET"]},
+        {"name": "storage.object_acl.delete",             "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.object_acl.insert",             "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.object_acl.patch",              "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.object_acl.update",             "resources": ["BUCKET", "OBJECT"]}
+      ],
+      "preconditionProvided": false,
+      "expectSuccess": false
+    },
+    {
+      "id": 5,
+      "description": "non-retryable errors",
+      "cases": [
+        {
+          "instructions": ["return-400"]
+        },
+        {
+          "instructions": ["return-401"]
+        }
+      ],
+      "methods": [
+        {"name": "storage.bucket_acl.delete",             "resources": ["BUCKET"]},
+        {"name": "storage.bucket_acl.get",                "resources": ["BUCKET"]},
+        {"name": "storage.bucket_acl.insert",             "resources": ["BUCKET"]},
+        {"name": "storage.bucket_acl.list",               "resources": ["BUCKET"]},
+        {"name": "storage.bucket_acl.patch",              "resources": ["BUCKET"]},
+        {"name": "storage.bucket_acl.update",             "resources": ["BUCKET"]},
+        {"name": "storage.buckets.delete",                "resources": ["BUCKET"]},
+        {"name": "storage.buckets.get",                   "resources": ["BUCKET"]},
+        {"name": "storage.buckets.getIamPolicy",          "resources": ["BUCKET"]},
+        {"name": "storage.buckets.insert",                "resources": ["BUCKET"]},
+        {"name": "storage.buckets.list",                  "resources": ["BUCKET"]},
+        {"name": "storage.buckets.lockRetentionPolicy",   "resources": ["BUCKET"]},
+        {"name": "storage.buckets.patch",                 "resources": ["BUCKET"]},
+        {"name": "storage.buckets.setIamPolicy",          "resources": ["BUCKET"]},
+        {"name": "storage.buckets.testIamPermissions",    "resources": ["BUCKET"]},
+        {"name": "storage.buckets.update",                "resources": ["BUCKET"]},
+        {"name": "storage.default_object_acl.delete",     "resources": ["BUCKET"]},
+        {"name": "storage.default_object_acl.get",        "resources": ["BUCKET"]},
+        {"name": "storage.default_object_acl.insert",     "resources": ["BUCKET"]},
+        {"name": "storage.default_object_acl.list",       "resources": ["BUCKET"]},
+        {"name": "storage.default_object_acl.patch",      "resources": ["BUCKET"]},
+        {"name": "storage.default_object_acl.update",     "resources": ["BUCKET"]},
+        {"name": "storage.hmacKey.create",                "resources": []},
+        {"name": "storage.hmacKey.delete",                "resources": ["HMAC_KEY"]},
+        {"name": "storage.hmacKey.get",                   "resources": ["HMAC_KEY"]},
+        {"name": "storage.hmacKey.list",                  "resources": ["HMAC_KEY"]},
+        {"name": "storage.hmacKey.update",                "resources": ["HMAC_KEY"]},
+        {"name": "storage.notifications.delete",          "resources": ["BUCKET", "NOTIFICATION"]},
+        {"name": "storage.notifications.get",             "resources": ["BUCKET", "NOTIFICATION"]},
+        {"name": "storage.notifications.insert",          "resources": ["BUCKET", "NOTIFICATION"]},
+        {"name": "storage.notifications.list",            "resources": ["BUCKET", "NOTIFICATION"]},
+        {"name": "storage.object_acl.delete",             "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.object_acl.get",                "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.object_acl.insert",             "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.object_acl.list",               "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.object_acl.patch",              "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.object_acl.update",             "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.objects.compose",               "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.objects.copy",                  "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.objects.delete",                "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.objects.get",                   "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.objects.insert",                "resources": ["BUCKET"]},
+        {"name": "storage.objects.list",                  "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.objects.patch",                 "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.objects.rewrite",               "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.objects.update",                "resources": ["BUCKET", "OBJECT"]},
+        {"name": "storage.serviceaccount.get",            "resources": []}
+      ],
+      "preconditionProvided": false,
+      "expectSuccess": false
     }
   ]
 }
diff --git a/storage/retry_conformance_test.go b/storage/retry_conformance_test.go
index 894fb5fad..13ceea81e 100644
--- a/storage/retry_conformance_test.go
+++ b/storage/retry_conformance_test.go
@@ -19,6 +19,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
@@ -28,6 +30,7 @@ import (
 
 	"cloud.google.com/go/internal/uid"
 	storage_v1_tests "cloud.google.com/go/storage/internal/test/conformance"
+	"google.golang.org/api/iterator"
 	"google.golang.org/api/option"
 	raw "google.golang.org/api/storage/v1"
 	htransport "google.golang.org/api/transport/http"
@@ -50,13 +53,329 @@ type retryFunc func(ctx context.Context, c *Client, fs *resources, preconditions
 // wrap library methods that implement these calls. There may be multiple values
 // because multiple library methods may use the same call (e.g. get could be a
 // read or just a metadata get).
+//
+// There may be missing methods with respect to the json API as not all methods
+// are used in the client library. The following are not used:
+// storage.bucket_acl.get
+// storage.bucket_acl.insert
+// storage.bucket_acl.patch
+// storage.buckets.update
+// storage.default_object_acl.get
+// storage.default_object_acl.insert
+// storage.default_object_acl.patch
+// storage.notifications.get
+// storage.object_acl.get
+// storage.object_acl.insert
+// storage.object_acl.patch
+// storage.objects.copy
+// storage.objects.update
 var methods = map[string][]retryFunc{
+	// Idempotent operations
+	"storage.bucket_acl.list": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			_, err := c.Bucket(fs.bucket.Name).ACL().List(ctx)
+			return err
+		},
+	},
+	"storage.buckets.delete": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			// Delete files from bucket before deleting bucket
+			it := c.Bucket(fs.bucket.Name).Objects(ctx, nil)
+			for {
+				attrs, err := it.Next()
+				if err == iterator.Done {
+					break
+				}
+				if err != nil {
+					return err
+				}
+				if err := c.Bucket(fs.bucket.Name).Object(attrs.Name).Delete(ctx); err != nil {
+					return err
+				}
+			}
+			return c.Bucket(fs.bucket.Name).Delete(ctx)
+		},
+	},
 	"storage.buckets.get": {
 		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
 			_, err := c.Bucket(fs.bucket.Name).Attrs(ctx)
 			return err
 		},
 	},
+	"storage.buckets.getIamPolicy": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			_, err := c.Bucket(fs.bucket.Name).IAM().Policy(ctx)
+			return err
+		},
+	},
+	"storage.buckets.insert": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			return c.Bucket("bucket").Create(ctx, projectID, nil)
+		},
+	},
+	"storage.buckets.list": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			it := c.Buckets(ctx, projectID)
+			for {
+				_, err := it.Next()
+				if err == iterator.Done {
+					return nil
+				}
+				if err != nil {
+					return err
+				}
+			}
+		},
+	},
+	"storage.buckets.lockRetentionPolicy": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			// buckets.lockRetentionPolicy is always idempotent, but is a special case because IfMetagenerationMatch is always required
+			return c.Bucket(fs.bucket.Name).If(BucketConditions{MetagenerationMatch: fs.bucket.MetaGeneration}).LockRetentionPolicy(ctx)
+		},
+	},
+	"storage.buckets.testIamPermissions": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			_, err := c.Bucket(fs.bucket.Name).IAM().TestPermissions(ctx, nil)
+			return err
+		},
+	},
+	"storage.default_object_acl.list": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			_, err := c.Bucket(fs.bucket.Name).DefaultObjectACL().List(ctx)
+			return err
+		},
+	},
+	"storage.hmacKey.delete": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			// key must be inactive to delete:
+			c.HMACKeyHandle(projectID, fs.hmacKey.AccessID).Update(ctx, HMACKeyAttrsToUpdate{State: "INACTIVE"})
+			return c.HMACKeyHandle(projectID, fs.hmacKey.AccessID).Delete(ctx)
+		},
+	},
+	"storage.hmacKey.get": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			_, err := c.HMACKeyHandle(projectID, fs.hmacKey.AccessID).Get(ctx)
+			return err
+		},
+	},
+	"storage.hmacKey.list": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			it := c.ListHMACKeys(ctx, projectID)
+			for {
+				_, err := it.Next()
+				if err == iterator.Done {
+					return nil
+				}
+				if err != nil {
+					return err
+				}
+			}
+		},
+	},
+	"storage.notifications.delete": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			return c.Bucket(fs.bucket.Name).DeleteNotification(ctx, fs.notification.ID)
+		},
+	},
+	"storage.notifications.list": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			_, err := c.Bucket(fs.bucket.Name).Notifications(ctx)
+			return err
+		},
+	},
+	"storage.object_acl.list": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			_, err := c.Bucket(fs.bucket.Name).Object(fs.object.Name).ACL().List(ctx)
+			return err
+		},
+	},
+	"storage.objects.get": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			_, err := c.Bucket(fs.bucket.Name).Object(fs.object.Name).Attrs(ctx)
+			return err
+		},
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			r, err := c.Bucket(fs.bucket.Name).Object(fs.object.Name).NewReader(ctx)
+			if err != nil {
+				return err
+			}
+			_, err = io.Copy(ioutil.Discard, r)
+			return err
+		},
+	},
+	"storage.objects.list": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			it := c.Bucket(fs.bucket.Name).Objects(ctx, nil)
+			for {
+				_, err := it.Next()
+				if err == iterator.Done {
+					return nil
+				}
+				if err != nil {
+					return err
+				}
+			}
+		},
+	},
+	"storage.serviceaccount.get": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			_, err := c.ServiceAccount(ctx, projectID)
+			return err
+		},
+	},
+	// Conditionally idempotent operations
+	// (all conditionally idempotent operations currently fail)
+	"storage.buckets.patch": {
+		func(ctx context.Context, c *Client, fs *resources, preconditions bool) error {
+			uattrs := BucketAttrsToUpdate{StorageClass: "ARCHIVE"}
+			bkt := c.Bucket(fs.bucket.Name)
+			if preconditions {
+				bkt = c.Bucket(fs.bucket.Name).If(BucketConditions{MetagenerationMatch: fs.bucket.MetaGeneration})
+			}
+			_, err := bkt.Update(ctx, uattrs)
+			return err
+		},
+	},
+	"storage.buckets.setIamPolicy": {
+		func(ctx context.Context, c *Client, fs *resources, preconditions bool) error {
+			bkt := c.Bucket(fs.bucket.Name)
+			policy, err := bkt.IAM().Policy(ctx)
+			if err != nil {
+				return err
+			}
+
+			if err := bkt.IAM().SetPolicy(ctx, policy); err != nil {
+				return err
+			}
+			return fmt.Errorf("Etag preconditions not supported")
+		},
+	},
+	"storage.hmacKey.update": {
+		func(ctx context.Context, c *Client, fs *resources, preconditions bool) error {
+			key := c.HMACKeyHandle(projectID, fs.hmacKey.AccessID)
+
+			_, err := key.Update(ctx, HMACKeyAttrsToUpdate{State: "INACTIVE"})
+			if err != nil {
+				return err
+			}
+			return fmt.Errorf("Etag preconditions not supported")
+		},
+	},
+	"storage.objects.compose": {
+		func(ctx context.Context, c *Client, fs *resources, preconditions bool) error {
+			dstName := "new-object"
+			src := c.Bucket(fs.bucket.Name).Object(fs.object.Name)
+			dst := c.Bucket(fs.bucket.Name).Object(dstName)
+
+			if preconditions {
+				dst = c.Bucket(fs.bucket.Name).Object(dstName).If(Conditions{DoesNotExist: true})
+			}
+
+			_, err := dst.ComposerFrom(src).Run(ctx)
+			return err
+		},
+	},
+	"storage.objects.delete": {
+		func(ctx context.Context, c *Client, fs *resources, preconditions bool) error {
+			obj := c.Bucket(fs.bucket.Name).Object(fs.object.Name)
+
+			if preconditions {
+				obj = c.Bucket(fs.bucket.Name).Object(fs.object.Name).If(Conditions{GenerationMatch: fs.object.Generation})
+			}
+			return obj.Delete(ctx)
+		},
+	},
+	"storage.objects.insert": {
+		func(ctx context.Context, c *Client, fs *resources, preconditions bool) error {
+			obj := c.Bucket(fs.bucket.Name).Object("new-object.txt")
+
+			if preconditions {
+				obj = obj.If(Conditions{DoesNotExist: true})
+			}
+
+			objW := obj.NewWriter(ctx)
+			if _, err := io.Copy(objW, strings.NewReader("object body")); err != nil {
+				return fmt.Errorf("io.Copy: %v", err)
+			}
+			if err := objW.Close(); err != nil {
+				return fmt.Errorf("Writer.Close: %v", err)
+			}
+			return nil
+		},
+	},
+	"storage.objects.patch": {
+		func(ctx context.Context, c *Client, fs *resources, preconditions bool) error {
+			uattrs := ObjectAttrsToUpdate{Metadata: map[string]string{"foo": "bar"}}
+			obj := c.Bucket(fs.bucket.Name).Object(fs.object.Name)
+			if preconditions {
+				obj = obj.If(Conditions{MetagenerationMatch: fs.object.Metageneration})
+			}
+			_, err := obj.Update(ctx, uattrs)
+			return err
+		},
+	},
+	"storage.objects.rewrite": {
+		func(ctx context.Context, c *Client, fs *resources, preconditions bool) error {
+			dstName := "new-object"
+			src := c.Bucket(fs.bucket.Name).Object(fs.object.Name)
+			dst := c.Bucket(fs.bucket.Name).Object(dstName)
+
+			if preconditions {
+				dst = c.Bucket(fs.bucket.Name).Object(dstName).If(Conditions{DoesNotExist: true})
+			}
+
+			_, err := dst.CopierFrom(src).Run(ctx)
+			return err
+		},
+	},
+	// Non-idempotent operations
+	"storage.bucket_acl.delete": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			return c.Bucket(fs.bucket.Name).ACL().Delete(ctx, AllUsers)
+		},
+	},
+	"storage.bucket_acl.update": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			return c.Bucket(fs.bucket.Name).ACL().Set(ctx, AllAuthenticatedUsers, RoleOwner)
+		},
+	},
+	"storage.default_object_acl.delete": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			return c.Bucket(fs.bucket.Name).DefaultObjectACL().Delete(ctx, AllAuthenticatedUsers)
+		},
+	},
+	"storage.default_object_acl.update": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			return c.Bucket(fs.bucket.Name).DefaultObjectACL().Set(ctx, AllAuthenticatedUsers, RoleOwner)
+		},
+	},
+	"storage.hmacKey.create": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			_, err := c.CreateHMACKey(ctx, projectID, serviceAccountEmail)
+			return err
+		},
+	},
+	"storage.notifications.insert": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			notification := Notification{
+				TopicID:        "my-topic",
+				TopicProjectID: projectID,
+				PayloadFormat:  "json",
+			}
+			_, err := c.Bucket(fs.bucket.Name).AddNotification(ctx, &notification)
+			return err
+		},
+	},
+	"storage.object_acl.delete": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			return c.Bucket(fs.bucket.Name).Object(fs.object.Name).ACL().Delete(ctx, AllAuthenticatedUsers)
+		},
+	},
+	"storage.object_acl.update": {
+		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+			return c.Bucket(fs.bucket.Name).Object(fs.object.Name).ACL().Set(ctx, AllAuthenticatedUsers, RoleOwner)
+		},
+	},
 }
 
 func TestRetryConformance(t *testing.T) {