NOTES:
1. Oracle Golden Gate is installed on Amazon EC2 Instance which is in the same network as S3 Bucket
2. EC2 uses Assumed role to access the Bucket
3. AWSCLI can write objects to the bucket but only the Golden Gate Big Data Adapter fails.
I am reading on a post that the role needs "Create bucket" access. Is that true? Is there a way to over-ride this?
The company policy doesn't approve creating bucket through codes.
Any help on this is much appreciated.
Below are the error and config
-----------------------------------------------------------------------------------------------------------------
20-08-23 10:10:23 INFO OGG-03506 The source database character set, as determined from the trail file, is UTF-8.
=ERROR 2019-08-23 10:10:23.000202 [TaskEngine_1(FileFinalizeTask)] - Verify S3 bucket [bucket_name] failed.
com.amazonaws.services.s3.model.AmazonS3Exception: Access Denied (Service: Amazon S3; Status Code: 403; Error Code: AccessDenied; Request ID: 5EAFCFD994D96D7F; S3 Extended Request ID: T+zSE1uBIN1VJNj3MpVR3SbAkOnUEkrrtRaKrGvq3hdYOPoKTLJjw
Pf4lk4USDq7IA2v2ajfAas=)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1712) ~[aws-java-sdk-1.11.605.jar:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1367) ~[aws-java-sdk-1.11.605.jar:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1113) ~[aws-java-sdk-1.11.605.jar:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:770) ~[aws-java-sdk-1.11.605.jar:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:744) ~[aws-java-sdk-1.11.605.jar:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:726) ~[aws-java-sdk-1.11.605.jar:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:686) ~[aws-java-sdk-1.11.605.jar:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:668) ~[aws-java-sdk-1.11.605.jar:?]
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:532) ~[aws-java-sdk-1.11.605.jar:?]
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:512) ~[aws-java-sdk-1.11.605.jar:?]
at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4921) ~[aws-java-sdk-1.11.605.jar:?]
at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4867) ~[aws-java-sdk-1.11.605.jar:?]
at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4861) ~[aws-java-sdk-1.11.605.jar:?]
at com.amazonaws.services.s3.AmazonS3Client.listBuckets(AmazonS3Client.java:978) ~[aws-java-sdk-1.11.605.jar:?]
at com.amazonaws.services.s3.AmazonS3Client.listBuckets(AmazonS3Client.java:984) ~[aws-java-sdk-1.11.605.jar:?]
at oracle.goldengate.eventhandler.s3.S3BucketManager.initBucketList(S3BucketManager.java:78) ~[ggs3eh-12.3.2.1.1.005.jar:12.3.2.1.1.005]
at oracle.goldengate.eventhandler.s3.S3BucketManager.doesBucketExist(S3BucketManager.java:60) ~[ggs3eh-12.3.2.1.1.005.jar:12.3.2.1.1.005]
at oracle.goldengate.eventhandler.s3.S3BucketManager.verifyBucket(S3BucketManager.java:43) [ggs3eh-12.3.2.1.1.005.jar:12.3.2.1.1.005]
at oracle.goldengate.eventhandler.s3.S3Manager.upload(S3Manager.java:156) [ggs3eh-12.3.2.1.1.005.jar:12.3.2.1.1.005]
at oracle.goldengate.eventhandler.s3.S3EventHandler.execute(S3EventHandler.java:175) [ggs3eh-12.3.2.1.1.005.jar:12.3.2.1.1.005]
at oracle.goldengate.datasource.eventhandler.EventHandlerFramework.executeHandlers(EventHandlerFramework.java:73) [ggaddons-12.3.2.1.1.005.jar:12.3.2.1.1.005]
at oracle.goldengate.handler.filewriter.FileFinalizeManager$FileFinalizeTask.call(FileFinalizeManager.java:52) [ggfw-12.3.2.1.1.005.jar:12.3.2.1.1.005]
at oracle.goldengate.util.taskmanager.TaskPoolRunnable.run(TaskPoolRunnable.java:78) [ggutil-12.3.2.1.1.005.jar:12.3.2.1.1.005]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_212]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_212]
at oracle.goldengate.util.taskmanager.TaskPoolThread.run(TaskPoolThread.java:53) [ggutil-12.3.2.1.1.005.jar:12.3.2.1.1.005]
CONFIG
####The File Writer Handler
gg.handler.filewriter.type=filewriter
gg.handler.filewriter.mode=op
gg.handler.filewriter.pathMappingTemplate=./dirout
gg.handler.filewriter.stateFileDirectory=./dirsta
--gg.handler.filewriter.fileNameMappingTemplate=${fullyQualifiedTableName}_${currentTimestamp}.txt
gg.handler.filewriter.fileNameMappingTemplate=app-goldenGate-serviceportal_${schemaName}_${tableName}_${currentTimestamp}.avro
gg.handler.filewriter.fileRollInterval=15m
gg.handler.filewriter.finalizeAction=move-rename
gg.handler.filewriter.fileRenameMappingTemplate=app-goldenGate-serviceportal_${schemaName}_${tableName}_${currentTimestamp}_arc.avro
gg.handler.filewriter.movePathMappingTemplate=./dirdat/${schemaName}_${tableName}_${currentTimestamp[yyyy-MM-dd]}
gg.handler.filewriter.inactivityRollInterval=30s
gg.handler.filewriter.format=avro_row
gg.handler.filewriter.format.pkUpdateHandling=delete-insert
gg.handler.filewriter.includetokens=true
gg.handler.filewriter.partitionByTable=true
gg.handler.filewriter.rollOnShutdown=true
#The S3 Event Handler
gg.handler.filewriter.eventHandler=s3
gg.eventhandler.s3.type=S3
gg.eventhandler.s3.region=ap-southeast-2
gg.eventhandler.s3.bucketMappingTemplate=bucket_name
gg.eventhandler.s3.pathMappingTemplate=${schemaName}_{tableName}_${currentTimestamp[yyyy-MM-dd]}
gg.eventhandler.s3.finalizeAction=none
javawriter.stats.display=TRUE
javawriter.stats.full=TRUE
gg.log=log4j
#gg.log.level=DEBUG
gg.log.level=INFO
#gg.includeggtokens=true
gg.report.time=30sec
#Set the classpath here
gg.classpath=/home/oracle/aws_sdk/aws-java-sdk-1.11.605/lib/*:/home/oracle/aws_sdk/aws-java-sdk-1.11.605/third-party/lib/*:/home/oracle/aws_sdk/aws-java-sdk-1.11.605/third-party/lib/jackson-annotations-2.6.0.jar
gg.log=log4j
gg.log.level=DEBUG
--javawriter.bootoptions=-Xmx512m -Xms32m -Djava.class.path=.:ggjava/ggjava.jar -Daws.accessKeyId= -Daws.secretKey=
javawriter.bootoptions=-Xmx512m -Xms32m -Djava.class.path=.:ggjava/ggjava.jar