fix(storage): retry gRPC DEADLINE_EXCEEDED errors (#10635) · googleapis/google-cloud-go@0018415

@@ -33,6 +33,7 @@ import (

3333

"github.com/googleapis/gax-go/v2/callctx"

3434

"google.golang.org/api/iterator"

3535

"google.golang.org/grpc/codes"

36+

"google.golang.org/grpc/status"

3637

)

37383839

var emulatorClients map[string]storageClient

@@ -1415,6 +1416,43 @@ func TestRetryMaxAttemptsEmulated(t *testing.T) {

14151416

})

14161417

}

141714181419+

// Test that a timeout returns a DeadlineExceeded error, in spite of DeadlineExceeded being a retryable

1420+

// status when it is returned by the server.

1421+

func TestTimeoutErrorEmulated(t *testing.T) {

1422+

transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {

1423+

ctx, cancel := context.WithTimeout(context.Background(), time.Nanosecond)

1424+

defer cancel()

1425+

time.Sleep(5 * time.Nanosecond)

1426+

config := &retryConfig{backoff: &gax.Backoff{Initial: 10 * time.Millisecond}}

1427+

_, err := client.GetBucket(ctx, bucket, nil, idempotent(true), withRetryConfig(config))

1428+1429+

// Error may come through as a context.DeadlineExceeded (HTTP) or status.DeadlineExceeded (gRPC)

1430+

if !(errors.Is(err, context.DeadlineExceeded) || status.Code(err) == codes.DeadlineExceeded) {

1431+

t.Errorf("GetBucket: got unexpected error %v; want DeadlineExceeded", err)

1432+

}

1433+1434+

// Validate that error was not retried. If it was retried, that will be mentioned

1435+

// in the error string because of wrapping.

1436+

if strings.Contains(err.Error(), "retry") {

1437+

t.Errorf("GetBucket: got error %v, expected non-retried error", err)

1438+

}

1439+

})

1440+

}

1441+1442+

// Test that server-side DEADLINE_EXCEEDED errors are retried as expected with gRPC.

1443+

func TestRetryDeadlineExceedeEmulated(t *testing.T) {

1444+

transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {

1445+

ctx := context.Background()

1446+

instructions := map[string][]string{"storage.buckets.get": {"return-504", "return-504"}}

1447+

testID := createRetryTest(t, project, bucket, client, instructions)

1448+

ctx = callctx.SetHeaders(ctx, "x-retry-test-id", testID)

1449+

config := &retryConfig{maxAttempts: expectedAttempts(4), backoff: &gax.Backoff{Initial: 10 * time.Millisecond}}

1450+

if _, err := client.GetBucket(ctx, bucket, nil, idempotent(true), withRetryConfig(config)); err != nil {

1451+

t.Fatalf("GetBucket: got unexpected error %v, want nil", err)

1452+

}

1453+

})

1454+

}

1455+14181456

// createRetryTest creates a bucket in the emulator and sets up a test using the

14191457

// Retry Test API for the given instructions. This is intended for emulator tests

14201458

// of retry behavior that are not covered by conformance tests.