diff --git a/examples/v1/RetryClientDemo.java b/examples/v1/RetryClientDemo.java new file mode 100644 index 00000000..0e21bb3c --- /dev/null +++ b/examples/v1/RetryClientDemo.java @@ -0,0 +1,268 @@ +/* + * Authzed API examples for RetryableClient + */ +package v1; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +import com.authzed.api.v1.ConflictStrategy; +import com.authzed.api.v1.Core; +import com.authzed.api.v1.Core.ObjectReference; +import com.authzed.api.v1.Core.Relationship; +import com.authzed.api.v1.Core.SubjectReference; +import com.authzed.api.v1.RetryableClient; +import com.authzed.api.v1.SchemaServiceOuterClass; +import com.authzed.api.v1.SchemaServiceOuterClass.ReadSchemaRequest; +import com.authzed.api.v1.SchemaServiceOuterClass.ReadSchemaResponse; +import com.authzed.api.v1.SchemaServiceOuterClass.WriteSchemaRequest; +import com.authzed.api.v1.SchemaServiceOuterClass.WriteSchemaResponse; +import com.authzed.grpcutil.BearerToken; + +/** + * RetryClientDemo demonstrates using RetryableClient with different conflict strategies. + * + * This program connects to a local SpiceDB instance and imports relationships + * using each of the available conflict strategies: + * - FAIL: Returns an error if duplicate relationships are found + * - SKIP: Ignores duplicates and continues with import + * - TOUCH: Retries the import with TOUCH semantics for duplicates + */ +public class RetryClientDemo { + // SpiceDB connection details + private static final String SPICEDB_ADDRESS = "localhost:50051"; + private static final String PRESHARED_KEY = "foobar"; + + // Number of relationships to create in each test + private static final int RELATIONSHIPS_COUNT = 1000; + + public static void main(String[] args) { + System.out.println("RetryClientDemo: Demonstrating RetryableClient with different conflict strategies"); + + // Create a RetryableClient connected to SpiceDB + RetryableClient client = null; + try { + client = RetryableClient.newClient( + SPICEDB_ADDRESS, + new BearerToken(PRESHARED_KEY), + true); // Using plaintext connection + + // Write schema for document and user types + writeSchema(client); + + // Verify connection and read schema + verifyConnection(client); + + // Demonstrate each conflict strategy + demonstrateFailStrategy(client); + demonstrateSkipStrategy(client); + demonstrateTouchStrategy(client); + + System.out.println("\nDemo completed successfully!"); + } catch (Exception e) { + System.err.println("Error in RetryClientDemo: " + e.getMessage()); + e.printStackTrace(); + } finally { + if (client != null) { + client.close(); + } + } + } + + /** + * Write a schema to SpiceDB with document and user types. + */ + private static void writeSchema(RetryableClient client) { + System.out.println("Writing schema to SpiceDB..."); + + // Define a schema with document and user types + String schema = """ + definition document { + relation reader: user + relation writer: user + + permission read = reader + writer + permission write = writer + } + + definition user {} + """; + + // Build the write schema request + WriteSchemaRequest request = WriteSchemaRequest.newBuilder() + .setSchema(schema) + .build(); + + try { + // Write the schema + WriteSchemaResponse response = client.schemaService() + .withDeadlineAfter(5, TimeUnit.SECONDS) + .writeSchema(request); + + System.out.println("Schema written successfully!"); + } catch (Exception e) { + System.err.println("Failed to write schema: " + e.getMessage()); + throw new RuntimeException("Could not write schema to SpiceDB", e); + } + } + + /** + * Verify connection to SpiceDB by reading the schema. + */ + private static void verifyConnection(RetryableClient client) { + try { + ReadSchemaResponse response = client.schemaService() + .withDeadlineAfter(5, TimeUnit.SECONDS) + .readSchema(ReadSchemaRequest.newBuilder().build()); + + System.out.println("\nSuccessfully connected to SpiceDB!"); + System.out.println("Schema: " + response.getSchemaText()); + } catch (Exception e) { + System.err.println("Failed to connect to SpiceDB: " + e.getMessage()); + throw new RuntimeException("Could not connect to SpiceDB", e); + } + } + + /** + * Demonstrate FAIL conflict strategy. + * This strategy will fail if duplicate relationships are found. + */ + private static void demonstrateFailStrategy(RetryableClient client) { + System.out.println("\n=== Demonstrating FAIL Strategy ==="); + try { + // Create unique relationships + List relationships = generateUniqueRelationships(RELATIONSHIPS_COUNT); + + System.out.println("Importing " + relationships.size() + " unique relationships with FAIL strategy..."); + long numLoaded = client.retryableBulkImportRelationships(relationships, ConflictStrategy.FAIL); + System.out.println("Successfully imported " + numLoaded + " relationships!"); + + // Now try with some duplicate relationships + try { + System.out.println("Now attempting to import same relationships again..."); + client.retryableBulkImportRelationships(relationships, ConflictStrategy.FAIL); + System.out.println("ERROR: Import should have failed but didn't!"); + } catch (Exception e) { + System.out.println("As expected, import failed with error: " + e.getMessage()); + } + } catch (Exception e) { + System.err.println("Error demonstrating FAIL strategy: " + e.getMessage()); + e.printStackTrace(); + } + } + + /** + * Demonstrate SKIP conflict strategy. + * This strategy will ignore duplicates and continue with the import. + */ + private static void demonstrateSkipStrategy(RetryableClient client) { + System.out.println("\n=== Demonstrating SKIP Strategy ==="); + try { + // Create a mix of new and existing relationships + List mixedRelationships = generateMixedRelationships(RELATIONSHIPS_COUNT / 2); + + System.out.println("Importing " + mixedRelationships.size() + " relationships (mix of new and existing) with SKIP strategy..."); + long numLoaded = client.retryableBulkImportRelationships(mixedRelationships, ConflictStrategy.SKIP); + + System.out.println("Successfully processed " + numLoaded + " relationships with SKIP strategy!"); + System.out.println("Note: Duplicates were skipped, but operation completed successfully"); + } catch (Exception e) { + System.err.println("Error demonstrating SKIP strategy: " + e.getMessage()); + e.printStackTrace(); + } + } + + /** + * Demonstrate TOUCH conflict strategy. + * This strategy will retry the import with TOUCH semantics for duplicates. + */ + private static void demonstrateTouchStrategy(RetryableClient client) { + System.out.println("\n=== Demonstrating TOUCH Strategy ==="); + try { + // Create all new relationships to ensure initial write works + List newRelationships = generateUniqueRelationships(RELATIONSHIPS_COUNT / 2, RELATIONSHIPS_COUNT); + + System.out.println("Importing " + newRelationships.size() + " new relationships..."); + long numLoaded = client.retryableBulkImportRelationships(newRelationships, ConflictStrategy.TOUCH); + System.out.println("Successfully imported " + numLoaded + " relationships!"); + + // Now use TOUCH on a mix of new and existing + List mixedRelationships = new ArrayList<>(newRelationships); + mixedRelationships.addAll(generateUniqueRelationships(RELATIONSHIPS_COUNT / 4, RELATIONSHIPS_COUNT * 2)); + + System.out.println("Now importing " + mixedRelationships.size() + " relationships with some duplicates using TOUCH strategy..."); + numLoaded = client.retryableBulkImportRelationships(mixedRelationships, ConflictStrategy.TOUCH); + + System.out.println("Successfully processed " + numLoaded + " relationships with TOUCH strategy!"); + System.out.println("Note: Duplicates were touched (re-written) rather than causing an error"); + } catch (Exception e) { + System.err.println("Error demonstrating TOUCH strategy: " + e.getMessage()); + e.printStackTrace(); + } + } + + /** + * Generate a list of unique relationships. + */ + private static List generateUniqueRelationships(int count) { + return generateUniqueRelationships(count, 0); + } + + /** + * Generate a list of unique relationships with IDs starting from offset. + */ + private static List generateUniqueRelationships(int count, int offset) { + List relationships = new ArrayList<>(count); + Random random = new Random(); + + for (int i = 0; i < count; i++) { + String docId = "doc" + (i + offset); + String userId = "user" + (random.nextInt(20) + 1); // 20 possible users + String relation = random.nextBoolean() ? "reader" : "writer"; + + relationships.add(createRelationship(docId, relation, userId)); + } + + return relationships; + } + + /** + * Generate a mix of new and potentially duplicate relationships. + */ + private static List generateMixedRelationships(int count) { + List relationships = new ArrayList<>(count); + Random random = new Random(); + + for (int i = 0; i < count; i++) { + // Use a lower document ID range to increase chance of duplicates + String docId = "doc" + (random.nextInt(count / 2) + 1); + String userId = "user" + (random.nextInt(10) + 1); + String relation = random.nextBoolean() ? "reader" : "writer"; + + relationships.add(createRelationship(docId, relation, userId)); + } + + return relationships; + } + + /** + * Create a relationship between a document and user with the specified relation. + */ + private static Relationship createRelationship(String docId, String relation, String userId) { + return Relationship.newBuilder() + .setResource(ObjectReference.newBuilder() + .setObjectType("document") + .setObjectId(docId) + .build()) + .setRelation(relation) + .setSubject(SubjectReference.newBuilder() + .setObject(ObjectReference.newBuilder() + .setObjectType("user") + .setObjectId(userId) + .build()) + .build()) + .build(); + } +} \ No newline at end of file diff --git a/src/intTest/java/SimpleRetryableClientTest.java b/src/intTest/java/SimpleRetryableClientTest.java new file mode 100644 index 00000000..f78ea432 --- /dev/null +++ b/src/intTest/java/SimpleRetryableClientTest.java @@ -0,0 +1,64 @@ +import java.util.ArrayList; +import java.util.List; + +import org.junit.Test; + +import com.authzed.api.v1.ConflictStrategy; +import com.authzed.api.v1.ObjectReference; +import com.authzed.api.v1.Relationship; +import com.authzed.api.v1.RetryableClient; +import com.authzed.api.v1.SubjectReference; +import com.authzed.grpcutil.BearerToken; + +import static org.junit.Assert.assertEquals; + +/** + * Simple test for RetryableClient that doesn't use mocking. + * This allows us to test the basic compilation and functionality. + */ +public class SimpleRetryableClientTest { + + @Test + public void testRetryableClientInitialization() { + // Create a real RetryableClient + RetryableClient client = RetryableClient.newClient( + "localhost:50051", + new BearerToken("test-token"), + true); + + // If we can create the client without errors, the test passes + client.close(); + } + + @Test + public void testCreateRelationship() { + // Create a relationship + Relationship relationship = createTestRelationship(); + + // Just verify the relationship object was created correctly + assertEquals("document", relationship.getResource().getObjectType()); + assertEquals("doc1", relationship.getResource().getObjectId()); + assertEquals("viewer", relationship.getRelation()); + assertEquals("user", relationship.getSubject().getObject().getObjectType()); + assertEquals("user1", relationship.getSubject().getObject().getObjectId()); + } + + /** + * Helper method to create a test relationship. + */ + private Relationship createTestRelationship() { + return Relationship.newBuilder() + .setResource(ObjectReference.newBuilder() + .setObjectType("document") + .setObjectId("doc1") + .build()) + .setRelation("viewer") + .setSubject(SubjectReference.newBuilder() + .setObject(ObjectReference.newBuilder() + .setObjectType("user") + .setObjectId("user1") + .build()) + .build()) + .build(); + } +} \ No newline at end of file diff --git a/src/main/java/com/authzed/api/v1/ConflictStrategy.java b/src/main/java/com/authzed/api/v1/ConflictStrategy.java new file mode 100644 index 00000000..da996c2e --- /dev/null +++ b/src/main/java/com/authzed/api/v1/ConflictStrategy.java @@ -0,0 +1,22 @@ +package com.authzed.api.v1; + +/** + * ConflictStrategy represents the strategy to be used when a conflict occurs + * during a bulk import of relationships into SpiceDB. + */ +public enum ConflictStrategy { + /** + * FAIL - The operation will fail if any duplicate relationships are found. + */ + FAIL, + + /** + * SKIP - The operation will ignore duplicates and continue with the import. + */ + SKIP, + + /** + * TOUCH - The operation will retry the import with TOUCH semantics in case of duplicates. + */ + TOUCH +} \ No newline at end of file diff --git a/src/main/java/com/authzed/api/v1/RetryableClient.java b/src/main/java/com/authzed/api/v1/RetryableClient.java new file mode 100644 index 00000000..ecaad767 --- /dev/null +++ b/src/main/java/com/authzed/api/v1/RetryableClient.java @@ -0,0 +1,499 @@ +package com.authzed.api.v1; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +import com.authzed.api.v1.ExperimentalServiceGrpc.ExperimentalServiceBlockingStub; +import com.authzed.api.v1.ExperimentalServiceGrpc.ExperimentalServiceStub; +import com.authzed.api.v1.PermissionsServiceGrpc.PermissionsServiceBlockingStub; +import com.authzed.api.v1.PermissionsServiceGrpc.PermissionsServiceStub; +import com.authzed.api.v1.SchemaServiceGrpc.SchemaServiceBlockingStub; + +import io.grpc.Channel; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.CallCredentials; +import io.grpc.Metadata; +import io.grpc.stub.StreamObserver; + +/** + * RetryableClient represents an open connection to SpiceDB with + * experimental services available. It also adds methods for + * retrying bulk imports with different conflict strategies. + * + * Clients are backed by a gRPC client and as such are thread-safe. + */ +public class RetryableClient { + private static final Logger logger = Logger.getLogger(RetryableClient.class.getName()); + + // Default configuration + private static final long DEFAULT_BACKOFF_MS = 50; + private static final long DEFAULT_MAX_RETRIES = 10; + private static final long DEFAULT_MAX_BACKOFF_MS = 2000; + private static final int DEFAULT_TIMEOUT_SECONDS = 30; + + // Fallback for datastore implementations on SpiceDB < 1.29.0 not returning proper gRPC codes + private static final List TX_CONFLICT_STRINGS = Arrays.asList( + "SQLSTATE 23505", // CockroachDB + "Error 1062 (23000)" // MySQL + ); + + private static final List RETRYABLE_ERROR_STRINGS = Arrays.asList( + "retryable error", // CockroachDB, PostgreSQL + "try restarting transaction", "Error 1205" // MySQL + ); + + private final SchemaServiceGrpc.SchemaServiceBlockingStub schemaService; + private final PermissionsServiceGrpc.PermissionsServiceBlockingStub permissionsService; + private final PermissionsServiceGrpc.PermissionsServiceStub asyncPermissionsService; + private final ExperimentalServiceBlockingStub experimentalService; + private final ExperimentalServiceStub asyncExperimentalService; + private final ManagedChannel channel; + + /** + * Create a new RetryableClient with the specified endpoint and credentials. + * + * @param target The endpoint to connect to + * @param credentials Call credentials for authentication + * @return A new RetryableClient instance + */ + public static RetryableClient newClient(String target, CallCredentials credentials) { + return newClient(target, credentials, false); + } + + /** + * Create a new RetryableClient with the specified endpoint, credentials and TLS settings. + * + * @param target The endpoint to connect to + * @param credentials Call credentials for authentication + * @param usePlaintext If true, use plaintext connection (no TLS) + * @return A new RetryableClient instance + */ + public static RetryableClient newClient(String target, CallCredentials credentials, boolean usePlaintext) { + ManagedChannelBuilder builder = ManagedChannelBuilder.forTarget(target); + if (usePlaintext) { + builder.usePlaintext(); + } else { + builder.useTransportSecurity(); + } + + ManagedChannel channel = builder.build(); + return new RetryableClient(channel, credentials); + } + + /** + * Create a new RetryableClient with the specified channel and credentials. + * + * @param channel The gRPC channel to use + * @param credentials Call credentials for authentication + */ + public RetryableClient(Channel channel, CallCredentials credentials) { + if (channel instanceof ManagedChannel) { + this.channel = (ManagedChannel) channel; + } else { + this.channel = null; + } + + this.schemaService = SchemaServiceGrpc.newBlockingStub(channel) + .withCallCredentials(credentials); + this.permissionsService = PermissionsServiceGrpc.newBlockingStub(channel) + .withCallCredentials(credentials); + this.asyncPermissionsService = PermissionsServiceGrpc.newStub(channel) + .withCallCredentials(credentials); + this.experimentalService = ExperimentalServiceGrpc.newBlockingStub(channel) + .withCallCredentials(credentials); + this.asyncExperimentalService = ExperimentalServiceGrpc.newStub(channel) + .withCallCredentials(credentials); + } + + /** + * Get the schema service client. + * + * @return The schema service client + */ + public SchemaServiceGrpc.SchemaServiceBlockingStub schemaService() { + return schemaService; + } + + /** + * Get the permissions service client. + * + * @return The permissions service client + */ + public PermissionsServiceGrpc.PermissionsServiceBlockingStub permissionsService() { + return permissionsService; + } + + /** + * Get the async permissions service client. + * + * @return The async permissions service client + */ + public PermissionsServiceGrpc.PermissionsServiceStub asyncPermissionsService() { + return asyncPermissionsService; + } + + /** + * Get the experimental service client. + * + * @return The experimental service client + */ + public ExperimentalServiceBlockingStub experimentalService() { + return experimentalService; + } + + /** + * Get the async experimental service client. + * + * @return The async experimental service client + */ + public ExperimentalServiceStub asyncExperimentalService() { + return asyncExperimentalService; + } + + /** + * Close the client connection. + */ + public void close() { + if (channel != null) { + try { + channel.shutdown().awaitTermination(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + logger.log(Level.WARNING, "Error shutting down channel", e); + Thread.currentThread().interrupt(); + } + } + } + + /** + * RetryableBulkImportRelationships is a wrapper around BulkImportRelationships. + * It retries the bulk import with different conflict strategies in case of failure. + * + * The conflict strategy can be one of: + * - FAIL - will return an error if any duplicate relationships are found. + * - SKIP - will ignore duplicates and continue with the import. + * - TOUCH - will retry the import with TOUCH semantics in case of duplicates. + * + * @param relationships The relationships to import + * @param conflictStrategy The conflict strategy to use + * @return The number of relationships loaded + * @throws Exception If there are errors during import + */ + public long retryableBulkImportRelationships( + List relationships, + ConflictStrategy conflictStrategy) throws Exception { + return retryableBulkImportRelationships(relationships, conflictStrategy, DEFAULT_TIMEOUT_SECONDS); + } + + /** + * RetryableBulkImportRelationships is a wrapper around BulkImportRelationships. + * It retries the bulk import with different conflict strategies in case of failure. + * + * The conflict strategy can be one of: + * - FAIL - will return an error if any duplicate relationships are found. + * - SKIP - will ignore duplicates and continue with the import. + * - TOUCH - will retry the import with TOUCH semantics in case of duplicates. + * + * @param relationships The relationships to import + * @param conflictStrategy The conflict strategy to use + * @param timeoutSeconds The timeout for the operation in seconds + * @return The number of relationships loaded + * @throws Exception If there are errors during import + */ + public long retryableBulkImportRelationships( + List relationships, + ConflictStrategy conflictStrategy, + int timeoutSeconds) throws Exception { + + final BulkImportResponseCollector responseCollector = new BulkImportResponseCollector(); + final StreamObserver responseObserver = new StreamObserver() { + @Override + public void onNext(BulkImportRelationshipsResponse response) { + responseCollector.addResponse(response); + } + + @Override + public void onError(Throwable t) { + responseCollector.setError(t); + } + + @Override + public void onCompleted() { + responseCollector.setCompleted(); + } + }; + + // Create initial request + BulkImportRelationshipsRequest request = BulkImportRelationshipsRequest.newBuilder() + .addAllRelationships(relationships) + .build(); + + // Send the request + StreamObserver requestObserver = + asyncExperimentalService.bulkImportRelationships(responseObserver); + + try { + requestObserver.onNext(request); + requestObserver.onCompleted(); + + // Wait for response to complete + BulkImportRelationshipsResponse response = responseCollector.await(); + return response.getNumLoaded(); + } catch (Throwable throwable) { + logger.log(Level.INFO, "Handling error in retryableBulkImportRelationships: " + throwable.getMessage()); + + // Handle the error based on its type and the chosen conflict strategy + if (isCanceledError(throwable)) { + throw new RuntimeException("Request canceled", throwable); + } + + if (isAlreadyExistsError(throwable)) { + if (conflictStrategy == ConflictStrategy.SKIP) { + // Skip conflicts - return success + logger.log(Level.INFO, "ALREADY_EXISTS detected with SKIP strategy - returning success"); + return relationships.size(); + } else if (conflictStrategy == ConflictStrategy.TOUCH) { + // Retry with touch semantics + logger.log(Level.INFO, "ALREADY_EXISTS detected with TOUCH strategy - retrying with touch semantics"); + return writeBatchesWithRetry(relationships, timeoutSeconds); + } else if (conflictStrategy == ConflictStrategy.FAIL) { + throw new RuntimeException("Duplicate relationships found", throwable); + } + } + + if (isRetryableError(throwable)) { + // Retry with touch semantics for retryable errors regardless of strategy + logger.log(Level.INFO, "Retryable error detected - retrying with touch semantics"); + return writeBatchesWithRetry(relationships, timeoutSeconds); + } + + throw new RuntimeException( + String.format("Error finalizing write of %d relationships", relationships.size()), + throwable); + } + } + + /** + * Helper class to collect response from bulk import. + */ + private static class BulkImportResponseCollector { + private BulkImportRelationshipsResponse response; + private Throwable error; + private boolean completed = false; + + synchronized void addResponse(BulkImportRelationshipsResponse resp) { + response = resp; + } + + synchronized void setError(Throwable t) { + error = t; + completed = true; + notifyAll(); + } + + synchronized void setCompleted() { + completed = true; + notifyAll(); + } + + synchronized BulkImportRelationshipsResponse await() throws Exception { + while (!completed) { + try { + wait(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Operation interrupted", e); + } + } + + if (error != null) { + throw new RuntimeException("Error in bulk import", error); + } + + return response; + } + } + + /** + * Retry writing relationships in batches with exponential backoff. + */ + private long writeBatchesWithRetry(List relationships, int timeoutSeconds) throws Exception { + long backoffMs = DEFAULT_BACKOFF_MS; + long currentRetries = 0; + + // Convert relationships to updates with TOUCH operation + List updates = new ArrayList<>(relationships.size()); + for (Relationship relationship : relationships) { + updates.add(RelationshipUpdate.newBuilder() + .setRelationship(relationship) + .setOperation(RelationshipUpdate.Operation.OPERATION_TOUCH) + .build()); + } + + while (true) { + try { + WriteRelationshipsRequest request = WriteRelationshipsRequest.newBuilder() + .addAllUpdates(updates) + .build(); + + WriteRelationshipsResponse response = permissionsService.withDeadlineAfter( + timeoutSeconds, TimeUnit.SECONDS).writeRelationships(request); + + return updates.size(); // Success + } catch (Exception e) { + if (isRetryableError(e) && currentRetries < DEFAULT_MAX_RETRIES) { + // Throttle the writes with exponential backoff + Thread.sleep(backoffMs); + backoffMs = Math.min(backoffMs * 2, DEFAULT_MAX_BACKOFF_MS); + currentRetries++; + continue; + } + + throw new RuntimeException("Failed to write relationships after retry", e); + } + } + } + + /** + * Check if an error indicates that a relationship already exists. + */ + private boolean isAlreadyExistsError(Throwable throwable) { + if (throwable == null) { + return false; + } + + // Check direct gRPC status + if (isGrpcStatusCode(throwable, Status.Code.ALREADY_EXISTS)) { + return true; + } + + // Check error message for "ALREADY_EXISTS" string + if (throwable.getMessage() != null && + throwable.getMessage().contains("ALREADY_EXISTS")) { + return true; + } + + // Check for transaction conflict strings + if (containsErrorString(throwable, TX_CONFLICT_STRINGS)) { + return true; + } + + // Check for a wrapped cause + Throwable cause = throwable.getCause(); + if (cause != null && cause != throwable) { + return isAlreadyExistsError(cause); + } + + return false; + } + + /** + * Check if an error is retryable. + */ + private boolean isRetryableError(Throwable throwable) { + if (throwable == null) { + return false; + } + + // Check direct gRPC status + if (isGrpcStatusCode(throwable, Status.Code.UNAVAILABLE, Status.Code.DEADLINE_EXCEEDED)) { + return true; + } + + // Check for retryable error strings + if (containsErrorString(throwable, RETRYABLE_ERROR_STRINGS)) { + return true; + } + + // Check for timeout exceptions + if (throwable instanceof java.util.concurrent.TimeoutException) { + return true; + } + + // Check for a wrapped cause + Throwable cause = throwable.getCause(); + if (cause != null && cause != throwable) { + return isRetryableError(cause); + } + + return false; + } + + /** + * Check if an error indicates that the request was canceled. + */ + private boolean isCanceledError(Throwable throwable) { + if (throwable == null) { + return false; + } + + // Check for cancellation exceptions + if (throwable instanceof java.util.concurrent.CancellationException || + throwable instanceof InterruptedException) { + return true; + } + + // Check direct gRPC status + if (isGrpcStatusCode(throwable, Status.Code.CANCELLED)) { + return true; + } + + // Check for a wrapped cause + Throwable cause = throwable.getCause(); + if (cause != null && cause != throwable) { + return isCanceledError(cause); + } + + return false; + } + + /** + * Check if the throwable's message contains any of the given error strings. + */ + private boolean containsErrorString(Throwable throwable, List errorStrings) { + if (throwable == null) { + return false; + } + + String message = throwable.getMessage(); + if (message == null) { + return false; + } + + for (String errorString : errorStrings) { + if (message.contains(errorString)) { + return true; + } + } + + return false; + } + + /** + * Check if the throwable is a gRPC error with one of the given status codes. + */ + private boolean isGrpcStatusCode(Throwable throwable, Status.Code... codes) { + if (throwable == null) { + return false; + } + + if (throwable instanceof StatusRuntimeException) { + StatusRuntimeException statusException = (StatusRuntimeException) throwable; + Status.Code throwableCode = statusException.getStatus().getCode(); + + for (Status.Code code : codes) { + if (code == throwableCode) { + return true; + } + } + } + + return false; + } +} \ No newline at end of file