fix(storage): retry gRPC DEADLINE_EXCEEDED errors (#10635) · googleapis/google-cloud-go@0018415
@@ -33,6 +33,7 @@ import (
3333"github.com/googleapis/gax-go/v2/callctx"
3434"google.golang.org/api/iterator"
3535"google.golang.org/grpc/codes"
36+"google.golang.org/grpc/status"
3637)
37383839var emulatorClients map[string]storageClient
@@ -1415,6 +1416,43 @@ func TestRetryMaxAttemptsEmulated(t *testing.T) {
14151416 })
14161417}
141714181419+// Test that a timeout returns a DeadlineExceeded error, in spite of DeadlineExceeded being a retryable
1420+// status when it is returned by the server.
1421+func TestTimeoutErrorEmulated(t *testing.T) {
1422+transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
1423+ctx, cancel := context.WithTimeout(context.Background(), time.Nanosecond)
1424+defer cancel()
1425+time.Sleep(5 * time.Nanosecond)
1426+config := &retryConfig{backoff: &gax.Backoff{Initial: 10 * time.Millisecond}}
1427+_, err := client.GetBucket(ctx, bucket, nil, idempotent(true), withRetryConfig(config))
1428+1429+// Error may come through as a context.DeadlineExceeded (HTTP) or status.DeadlineExceeded (gRPC)
1430+if !(errors.Is(err, context.DeadlineExceeded) || status.Code(err) == codes.DeadlineExceeded) {
1431+t.Errorf("GetBucket: got unexpected error %v; want DeadlineExceeded", err)
1432+ }
1433+1434+// Validate that error was not retried. If it was retried, that will be mentioned
1435+// in the error string because of wrapping.
1436+if strings.Contains(err.Error(), "retry") {
1437+t.Errorf("GetBucket: got error %v, expected non-retried error", err)
1438+ }
1439+ })
1440+}
1441+1442+// Test that server-side DEADLINE_EXCEEDED errors are retried as expected with gRPC.
1443+func TestRetryDeadlineExceedeEmulated(t *testing.T) {
1444+transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
1445+ctx := context.Background()
1446+instructions := map[string][]string{"storage.buckets.get": {"return-504", "return-504"}}
1447+testID := createRetryTest(t, project, bucket, client, instructions)
1448+ctx = callctx.SetHeaders(ctx, "x-retry-test-id", testID)
1449+config := &retryConfig{maxAttempts: expectedAttempts(4), backoff: &gax.Backoff{Initial: 10 * time.Millisecond}}
1450+if _, err := client.GetBucket(ctx, bucket, nil, idempotent(true), withRetryConfig(config)); err != nil {
1451+t.Fatalf("GetBucket: got unexpected error %v, want nil", err)
1452+ }
1453+ })
1454+}
1455+14181456// createRetryTest creates a bucket in the emulator and sets up a test using the
14191457// Retry Test API for the given instructions. This is intended for emulator tests
14201458// of retry behavior that are not covered by conformance tests.