Draft: passthrough in client - WIP

pom.xml

@@ -55,7 +55,7 @@
       <dependency>
         <groupId>software.amazon.awssdk</groupId>
         <artifactId>bom</artifactId>
-        <version>2.15.39</version>
+        <version>2.18.16</version>
         <type>pom</type>
         <scope>import</scope>
       </dependency>
@@ -69,15 +69,18 @@
     </dependency>
     <dependency>
       <groupId>software.amazon.awssdk</groupId>
-      <artifactId>sts</artifactId>
+      <artifactId>apache-client</artifactId>
     </dependency>
     <dependency>
-      <groupId>software.amazon.awssdk</groupId>
-      <artifactId>apache-client</artifactId>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>3.8.1</version>
+      <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>software.amazon.awssdk</groupId>
-      <artifactId>utils</artifactId>
+      <groupId>com.amazonaws</groupId>
+      <artifactId>aws-java-sdk-core</artifactId>
+      <version>1.11.615</version>
     </dependency>
   </dependencies>
 

src/main/java/com/gitlab/projectn_oss/bolt/BoltConfig.java

@@ -0,0 +1,104 @@
+package com.gitlab.projectn_oss.bolt;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import software.amazon.awssdk.regions.internal.util.EC2MetadataUtils;
+
+
+public class BoltConfig {
+
+    static String ReadOrderEndpoints[] = { "main_read_endpoints", "main_write_endpoints", "failover_read_endpoints", "failover_write_endpoints" };
+    static String WriteOrderEndpoints[] = { "main_write_endpoints", "failover_write_endpoints" };
+    static List<String> HttpReadMethodTypes = Arrays.asList( "GET", "HEAD" ); // S3 operations get converted to one of the standard HTTP request methods https://docs.aws.amazon.com/apigateway/latest/developerguide/integrating-api-with-aws-services-s3.html
+
+    public static String Region = System.getenv("AWS_REGION") == null
+        ? EC2MetadataUtils.getEC2InstanceRegion(): System.getenv("AWS_REGION");
+    public static String ZoneId = System.getenv("AWS_ZONE_ID") == null
+        ? EC2MetadataUtils.getAvailabilityZone(): System.getenv("AWS_ZONE_ID");
+    public static String CustomDomain = System.getenv("BOLT_CUSTOM_DOMAIN");
+    public static String AuthBucket = System.getenv("BOLT_AUTH_BUCKET");
+    static String BoltHostname = String.format("bolt.%s.%s", Region, CustomDomain);
+    static String QuicksilverUrl = String.format("https://quicksilver.%s.%s/services/bolt%s", Region, CustomDomain, ZoneId == null ? "": String.format("?az=%s", ZoneId));
+    //TODO: inplace of calling getBoltEndpoints, call refresh endpoints to avoid calling quicksilver everytime
+    static Map<String, List<String>> BoltEndpoints = getBoltEndpoints("");
+    public static Map<String, List<String>> getBoltEndpoints(String errIp){
+        if (QuicksilverUrl == null){
+            return null;
+        }
+        // BoltEndpoints
+        String requestUrl = errIp.length() > 0 ? 
+                String.format("%s&err=%s",QuicksilverUrl, errIp) : QuicksilverUrl;
+        try{
+            URL url = new URL(requestUrl);
+            HttpURLConnection con = (HttpURLConnection) url.openConnection();
+            con.setRequestMethod("GET");
+
+            int status = con.getResponseCode();
+
+            BufferedReader in = new BufferedReader(
+                new InputStreamReader(con.getInputStream()));
+            String inputLine;
+            StringBuffer content = new StringBuffer();
+            while ((inputLine = in.readLine()) != null) {
+                content.append(inputLine);
+            }
+            in.close();
+            con.disconnect();
+
+            return parse(content.toString());        
+        }
+        catch (Exception ex){
+            ex.printStackTrace();
+            return null;
+        }
+    }
+
+    private static Map<String, List<String>> parse(String responseBody) throws JsonMappingException, JsonProcessingException {
+        Map<String, List<String>> map = new HashMap<String, List<String>>();
+        ObjectMapper mapper = new ObjectMapper();
+        try {
+            map = mapper.readValue(responseBody, new TypeReference<HashMap<String, List<String>>>() {});
+        } catch (IOException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+        //TODO: Cleanup
+        return map;
+    }
+
+    public static URI selectBoltEndpoints(String httpRequestMethod){
+        if (BoltEndpoints == null){
+            BoltEndpoints = getBoltEndpoints("");
+        }
+        String[] preferredOrder = HttpReadMethodTypes.contains(httpRequestMethod) ? ReadOrderEndpoints : WriteOrderEndpoints;
+
+        // String[] endPointsKey;
+        for (String endPointsKey : preferredOrder){
+
+            if (BoltEndpoints.containsKey(endPointsKey) && BoltEndpoints.get(endPointsKey).size()>0){
+                //TODO: select a rendom index instate of 0
+                String selectedEndpoint = BoltEndpoints.get(endPointsKey).get(0);
+                try {
+                    return new URI(String.format("https://%s",selectedEndpoint));
+                } catch (URISyntaxException e) {
+                    // TODO Auto-generated catch block
+                    e.printStackTrace();
+                }
+            }
+        }
+        return null;
+    }
+}

src/main/java/com/gitlab/projectn_oss/bolt/BoltExecutionInterceptor.java

@@ -0,0 +1,72 @@
+package com.gitlab.projectn_oss.bolt;
+
+import software.amazon.awssdk.core.interceptor.*;
+import software.amazon.awssdk.http.SdkHttpRequest;
+import software.amazon.awssdk.http.SdkHttpResponse;
+import software.amazon.awssdk.core.SdkRequest;
+import software.amazon.awssdk.core.SdkResponse;
+import software.amazon.awssdk.core.ResponseInputStream;
+
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.util.Map;
+
+public class BoltExecutionInterceptor implements ExecutionInterceptor {
+
+    // The client which we will use to make requests to S3 (not Bolt)
+    private S3Client s3Client = S3Client.builder().build();
+
+    @Override
+    public SdkHttpResponse modifyHttpResponse(Context.ModifyHttpResponse context,
+                    ExecutionAttributes executionAttributes) {
+        System.out.println("BoltExecutionInterceptor: Hello from modifyHttpResponse");
+
+        // Print out all executionAttributes
+        Map<ExecutionAttribute<?>, Object> allAttrs = executionAttributes.getAttributes();
+        for (ExecutionAttribute ea: allAttrs.keySet() ) {
+            System.out.println("  " + ea + "->" + allAttrs.get(ea));
+        }
+
+//        return context.httpResponse();
+
+        SdkRequest req = context.request();
+        if (!(req instanceof GetObjectRequest)) {
+            System.out.println("request is NOT GetObject, do nothing");
+            return context.httpResponse();
+        }
+
+        System.out.println("request is GetObject");
+        System.out.println("  req: "+ req);
+
+        System.out.println("httpResponse: " + context.httpResponse());
+        System.out.println("  statusCode: " + context.httpResponse().statusCode());
+        System.out.println("  statusText: " + context.httpResponse().statusText());
+
+        System.out.println("Making call to real s3Client...");
+        GetObjectRequest getObjReq = (GetObjectRequest) req;
+        ResponseInputStream<GetObjectResponse> respInStream = s3Client.getObject(getObjReq);
+
+        BufferedReader reader = new BufferedReader(new InputStreamReader(respInStream));
+        String line;
+        try {
+            while ((line = reader.readLine()) != null) {
+                System.out.println("response-line: " + line);
+            }
+        } catch (IOException e) {
+            System.out.println("IOException: " + e);
+        }
+
+        SdkHttpResponse response = respInStream.response().sdkHttpResponse();
+
+        System.out.println("from real s3Client, httpResponse: " + response);
+        System.out.println("  statusCode: " + response.statusCode());
+        System.out.println("  statusText: " + response.statusText());
+
+        return response;
+    }
+}

src/main/java/com/gitlab/projectn_oss/bolt/BoltHostnameVerifier.java

@@ -0,0 +1,16 @@
+package com.gitlab.projectn_oss.bolt;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.SSLSession;
+import org.apache.http.conn.ssl.DefaultHostnameVerifier;
+
+public class BoltHostnameVerifier implements HostnameVerifier {
+
+    @Override
+    public boolean verify(String host, SSLSession sslSession) {
+        // TODO Auto-generated method stub
+        DefaultHostnameVerifier df = new DefaultHostnameVerifier();
+        return df.verify(BoltConfig.BoltHostname, sslSession);
+    }
+
+}

src/main/java/com/gitlab/projectn_oss/bolt/BoltS3Client.java

@@ -1,13 +1,21 @@
 package com.gitlab.projectn_oss.bolt;
 
+import javax.net.ssl.SSLContext;
+
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+
+import org.apache.http.conn.socket.ConnectionSocketFactory;
 import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
 import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption;
-import software.amazon.awssdk.regions.internal.util.EC2MetadataUtils;
+import software.amazon.awssdk.http.apache.ApacheHttpClient;
+import software.amazon.awssdk.http.apache.internal.conn.SdkTlsSocketFactory;
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.S3ClientBuilder;
 import software.amazon.awssdk.services.s3.S3Configuration;
 
-import java.net.URI;
+// import java.net.URI;
+
 
 /**
  * Service Client for accessing S3 via Bolt.
@@ -16,7 +24,7 @@
  */
 public interface BoltS3Client extends S3Client {
 
-    String BoltServiceUrl = System.getenv("BOLT_URL");
+    // Get bolt url from quicksilver. Not from the env
 
     /**
      * Creates a S3Client with the credentials loaded from the application's default configuration.
@@ -31,27 +39,31 @@ static S3Client create() {
      * @return S3ClientBuilder
      */
     static S3ClientBuilder builder() {
-        String BoltRegionalServiceUrl = BoltServiceUrl;
-        if (BoltRegionalServiceUrl.contains("{region}")) {
-            BoltRegionalServiceUrl = BoltRegionalServiceUrl.replace("{region}", Region());
+        SSLContext sslcontext = null;
+        try {
+            sslcontext = SSLContext.getInstance("TLS");
+        } catch (NoSuchAlgorithmException e1) {
+            System.out.println("SSL context is not initialized with error");
+            e1.printStackTrace();
+        }
+        try {
+            sslcontext.init(null, null, null);
+        } catch (KeyManagementException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
         }
+        ConnectionSocketFactory socketFactory = new SdkTlsSocketFactory(sslcontext, new BoltHostnameVerifier());
 
+        //TODO return error if boltconfig.CustomDomain is not set
         return S3Client.builder()
-                .endpointOverride(URI.create(BoltRegionalServiceUrl))
+                .httpClient(ApacheHttpClient.builder().socketFactory(socketFactory).build())
                 .serviceConfiguration(S3Configuration.builder()
                         .pathStyleAccessEnabled(true)
                         .build())
                 .overrideConfiguration(ClientOverrideConfiguration.builder()
                         .putAdvancedOption(SdkAdvancedClientOption.SIGNER, BoltSigner.create())
+                        .putHeader("X-Bolt-Passthrough-Read", "disable")
+                        .addExecutionInterceptor(new BoltExecutionInterceptor())
                         .build());
     }
-
-    static String Region() {
-        String region = System.getenv("AWS_REGION");
-        if (region != null) {
-            return region;
-        } else {
-            return EC2MetadataUtils.getEC2InstanceRegion();
-        }
-    }
 }