{"Event":"SparkListenerLogStart","Spark Version":"3.5.4-amzn-0"}
{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"executorType":{"Resource Name":"executorType","Amount":1,"Discovery Script":"","Vendor":""},"cores":{"Resource Name":"cores","Amount":4,"Discovery Script":"","Vendor":""},"memory":{"Resource Name":"memory","Amount":10240,"Discovery Script":"","Vendor":""},"offHeap":{"Resource Name":"offHeap","Amount":0,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}}
{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":1,"Executor Resource Requests":{"executorType":{"Resource Name":"executorType","Amount":1,"Discovery Script":"","Vendor":""},"cores":{"Resource Name":"cores","Amount":4,"Discovery Script":"","Vendor":""},"memory":{"Resource Name":"memory","Amount":10240,"Discovery Script":"","Vendor":""},"offHeap":{"Resource Name":"offHeap","Amount":0,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"172.36.69.93","Port":36995},"Maximum Memory":6253707264,"Timestamp":1761565571214,"Maximum Onheap Memory":6253707264,"Maximum Offheap Memory":0}
{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/java-17-amazon-corretto.x86_64","Java Version":"17.0.16 (Amazon.com Inc.)","Scala Version":"version 2.12.18"},"Spark Properties":{"spark.pyspark.python":"/usr/bin/python3.11","spark.python.dependencyVersion.pyparsing":"3.1.2","spark.glue.enable-observability-metrics":"true","spark.python.dependencyVersion.aiosignal":"1.3.1","spark.app.initial.archive.urls":"spark://172.36.69.93:46463/files/glue-job-14058754211823993811_glue_venv.zip#python_environment","spark.dynamicAllocation.shuffleTracking.enabled":"true","spark.python.dependencyVersion.aiohappyeyeballs":"2.3.5","spark.glue.additionalParams.ADDITIONAL_GLUE_JDK_OPTS":"-Xmx10g -XX:+UseG1GC -XX:MaxHeapFreeRatio=70 -XX:InitiatingHeapOccupancyPercent=45","spark.python.dependencyVersion.packaging":"24.1","spark.python.dependencyVersion.pyarrow":"17.0.0","spark.python.dependencyVersion.idna":"2.10","spark.python.dependencyVersion.appdirs":"1.4.4","spark.sql.parquet.fs.optimized.committer.optimization-enabled":"true","spark.hadoop.hive.metastore.client.factory.class":"com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory","spark.metrics.conf.*.sink.GlueCloudwatch.jobRunId":"jr_4464f784f631b1a0c7231da55ee1a6f368615377dff170f59beab638b8587351","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseParallelGC -XX:InitiatingHeapOccupancyPercent=70 -XX:OnOutOfMemoryError='kill -9 %p' -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseParallelGC -XX:InitiatingHeapOccupancyPercent=70 -XX:OnOutOfMemoryError='kill -9 %p'","spark.driver.host":"172.36.69.93","spark.serializer.objectStreamReset":"100","spark.history.fs.logDirectory":"file:///var/log/spark/apps","spark.python.dependencyVersion.pytz":"2024.1","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version.emr_internal_use_only.EmrFileSystem":"2","spark.python.dependencyVersion.cycler":"0.12.1","spark.hadoop.mapreduce.output.fs.optimized.committer.enabled":"true","spark.eventLog.enabled":"true","spark.glue.additionalParams.GLUE_LIBS_TEMP_DIR_PATH":"/tmp/glue-job-14058754211823993811/","spark.archives":"/tmp/glue-job-14058754211823993811_glue_venv.zip#python_environment","spark.hadoop.lakeformation.credentials.url":"http://localhost:9998/lakeformationcredentials","spark.python.dependencyVersion.pip":"22.3.1","spark.metrics.conf.driver.sink.AWSDILyra.class":"org.apache.spark.metrics.sink.AWSDILyraSink","spark.hadoop.fs.s3.impl":"com.amazon.ws.emr.hadoop.fs.EmrFileSystem","spark.python.dependencyVersion.fsspec":"2024.6.1","spark.hadoop.fs.s3a.committer.name":"magicv2","spark.python.dependencyVersion.fonttools":"4.53.1","spark.python.dependencyVersion.seaborn":"0.13.2","spark.glue.JOB_RUN_ID":"jr_4464f784f631b1a0c7231da55ee1a6f368615377dff170f59beab638b8587351","spark.metrics.conf.*.source.glue.verticalScaling.class":"org.apache.spark.metrics.source.VerticalScalingMetricsSource","spark.metrics.conf.driver.source.throughput.class":"org.apache.spark.metrics.source.ThroughputMetricsSource","spark.metrics.conf.driver.source.AWSDILyra.class":"org.apache.spark.metrics.source.AWSDILyraSource","spark.driver.port":"46463","spark.shuffle.service.enabled":"false","spark.emr-serverless.client.create.batch.size":"100","spark.glue.scriptName":"test_zodiac_pg_to_redshift.py","spark.extraListeners":"com.amazonaws.services.glueexceptionanalysis.GlueExceptionAnalysisListener,com.amazonaws.services.gluedi.sensei.autodebugging.listeners.AutoDebuggingListener,glue.ch.cern.sparkmeasure.FlightRecorderStageMetrics,glue.ch.cern.sparkmeasure.FlightRecorderTaskMetrics","spark.hadoop.parquet.enable.summary-metadata":"false","spark.rdd.compress":"True","spark.driver.extraLibraryPath":"/usr/lib/hadoop/lib/native:/usr/lib/hadoop-lzo/lib/native:/docker/usr/lib/hadoop/lib/native:/docker/usr/lib/hadoop-lzo/lib/native","spark.metrics.conf.*.sink.GlueCloudwatch.proxyHost":"null","spark.glue.enable-job-insights":"true","spark.glue.USE_PROXY":"false","spark.sparkmeasure.printToStdout":"false","spark.python.dependencyVersion.matplotlib":"3.9.0","spark.ui.custom.executor.log.url":"/logs/{{CONTAINER_ID}}/{{FILE_NAME}}.gz","spark.metrics.conf.*.source.jvm.class":"org.apache.spark.metrics.source.JvmSource","spark.executorEnv.PYTHONPATH":"python_environment","spark.python.dependencyVersion.amzn-awsgluelibs":"5.0.731","spark.jars":"/tmp/glue-job-14058754211823993811/jars/OBgZGQ-AwsGlueMLLibs.jar,/tmp/glue-job-14058754211823993811/jars/5qwJmK-aws-glue-di-package-5.0.731.jar","spark.hadoop.fs.defaultFS":"file:///","spark.python.dependencyVersion.numpy":"1.26.4","spark.metrics.conf.driver.source.glue.jobPerformance.class":"org.apache.spark.metrics.source.PerformanceMetricsSource","spark.python.dependencyVersion.filelock":"3.19.1","spark.python.dependencyVersion.aioitertools":"0.11.0","spark.glueAppInsightsLog.dir":"/tmp/glue-app-insights-logs/","spark.python.dependencyVersion.aiobotocore":"2.13.1","spark.stage.attempt.ignoreOnDecommissionFetchFailure":"true","spark.ui.enabled":"false","spark.python.dependencyVersion.python-dateutil":"2.9.0.post0","spark.app.name":"nativespark-test_zodiac_pg_to_redshift-jr_4464f784f631b1a0c7231da55ee1a6f368615377dff170f59beab638b8587351","spark.glue.accountId":"896271372615","spark.metrics.conf.driver.source.glue.resourceUtilization.class":"org.apache.spark.metrics.source.ResourceUtilizationMetricsSource","spark.glue.etl-container-telemetry-scraping-interval":"10s","spark.hadoop.fs.s3a.aws.credentials.provider":"software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider","spark.network.timeout":"600","spark.hadoop.fs.s3a.connection.maximum":"500","spark.glue.etl-enable-container-telemetry-collection":"true","spark.glue.additionalParams.PROXY_DISABLED":"false","spark.datasource.redshift.community.trace_id":"jr_4464f784f631b1a0c7231da55ee1a6f368615377dff170f59beab638b8587351","spark.metrics.conf.*.sink.GlueCloudwatch.proxyPort":"-1","spark.hadoop.fs.s3.buffer.dir":"/tmp/hadoop/s3","spark.glueExceptionAnalysisEventLog.dir":"/tmp/glue-exception-analysis-logs/","spark.scheduler.mode":"FIFO","spark.unsafe.sorter.spill.read.ahead.enabled":"false","spark.python.dependencyVersion.jmespath":"0.10.0","spark.python.dependencyVersion.distlib":"0.4.0","spark.python.dependencyVersion.uv":"0.6.14","spark.python.dependencyVersion.attrs":"24.2.0","spark.rpc.askTimeout":"600","spark.driver.memory":"10g","spark.executor.instances":"9","spark.python.version":"3.11.13","spark.python.dependencyVersion.aiohttp":"3.10.1","spark.glue.GLUE_VERSION":"5.0","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.python.dependencyVersion.certifi":"2024.7.4","spark.glue.GLUE_TASK_GROUP_ID":"3b0cded1-7f28-42f4-be98-ee53219e770c","spark.python.dependencyVersion.botocore":"1.34.131","spark.python.dependencyVersion.multidict":"6.0.5","spark.hadoop.fs.s3bfs.impl":"org.apache.hadoop.fs.s3.S3FileSystem","spark.sparkmeasure.outputFormat":"json","spark.driver.bindAddress":"172.36.69.93","spark.python.dependencyVersion.plotly":"5.23.0","spark.hadoop.mapred.output.direct.EmrFileSystem":"true","spark.python.dependencyVersion.kaleido":"0.2.1","spark.default.parallelism":"36","spark.glue.glue-libs-temp-dir-path":"/tmp/glue-job-14058754211823993811","spark.python.dependencyVersion.charset-normalizer":"3.3.2","spark.dynamicAllocation.maxExecutors":"9","spark.executor.defaultJavaOptions":"-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseParallelGC -XX:InitiatingHeapOccupancyPercent=70 -XX:OnOutOfMemoryError='kill -9 %p'","spark.glue.additionalParams.USER_JARS_FIRST":"false","spark.metrics.conf.*.source.system.class":"org.apache.spark.metrics.source.SystemMetricsSource","spark.resourceManager.cleanupExpiredHost":"true","spark.hadoop.fs.s3.maxConnections":"500","spark.app.startTime":"1761565568831","spark.executor.id":"driver","spark.python.dependencyVersion.pillow":"10.4.0","spark.sql.emr.internal.extensions":"com.amazonaws.emr.spark.EmrSparkSessionExtensions","spark.glueJobInsights.enabled":"true","spark.glue.GLUE_COMMAND_CRITERIA":"glueetl","spark.driver.cores":"4","spark.python.dependencyVersion.kiwisolver":"1.4.5","spark.submit.customResourceManager.submit.class":"org.apache.spark.deploy.emrserverless.submit.EmrServerlessClientApplication","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:OnOutOfMemoryError='kill -9 %p' -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false -XX:OnOutOfMemoryError='kill -9 %p'","spark.emr-serverless.client.release.batch.size":"100","spark.python.dependencyVersion.setuptools":"65.5.1","spark.python.dependencyVersion.contourpy":"1.2.1","spark.hadoop.fs.s3.getObject.initialSocketTimeoutMilliseconds":"2000","spark.hadoop.fs.s3a.committer.magic.track.commits.in.memory.enabled":"true","spark.app.initial.jar.urls":"spark://172.36.69.93:46463/jars/OBgZGQ-AwsGlueMLLibs.jar,spark://172.36.69.93:46463/jars/5qwJmK-aws-glue-di-package-5.0.731.jar","spark.submit.deployMode":"client","spark.metrics.conf.*.sink.GlueCloudwatch.accountId":"896271372615","spark.cloudwatch.logging.conf.jobRunId":"com.amazonaws.services.glue.launch.helpers.SparkPropsManager@24043ec5.LOGGING_JOB_RUN_ID","spark.hadoop.hive.metastore.warehouse.dir":"/tmp/spark-warehouse","spark.hadoop.mapreduce.fileoutputcommitter.marksuccessfuljobs":"false","spark.glue.additionalParams.ADDITIONAL_CLASSPATH":"*********(redacted)","spark.master":"custom:jes","spark.glue.glue-python-libs-dir":"/tmp/glue-job-14058754211823993811/python","spark.python.dependencyVersion.virtualenv":"20.4.0","spark.metrics.conf.driver.source.aggregate.class":"org.apache.spark.metrics.source.AggregateMetricsSource","spark.glue.glue-jars-dir":"/tmp/glue-job-14058754211823993811/jars","spark.sql.parquet.output.committer.class":"com.amazon.emr.committer.EmrOptimizedSparkSqlParquetOutputCommitter","spark.hadoop.fs.s3a.committer.magic.enabled":"true","spark.hadoop.fs.s3n.impl":"com.amazon.ws.emr.hadoop.fs.EmrFileSystem","spark.driver.defaultJavaOptions":"-XX:OnOutOfMemoryError='kill -9 %p'","spark.python.dependencyVersion.s3fs":"2024.6.1","spark.python.dependencyVersion.urllib3":"1.25.10","spark.metrics.conf.driver.source.glue.jobPerformance.skewnessFactor":"5","spark.authenticate":"true","spark.python.dependencyVersion.boto3":"1.34.131","spark.files.overwrite":"true","spark.metrics.conf.*.sink.GlueCloudwatch.jobName":"test_zodiac_pg_to_redshift","spark.python.dependencyVersion.pandas":"2.2.2","spark.authenticate.secret":"*********(redacted)","spark.blacklist.decommissioning.timeout":"1h","spark.executor.extraLibraryPath":"/usr/lib/hadoop/lib/native:/usr/lib/hadoop-lzo/lib/native:/docker/usr/lib/hadoop/lib/native:/docker/usr/lib/hadoop-lzo/lib/native","spark.sql.hive.metastore.sharedPrefixes":"software.amazon.awssdk.services.dynamodb","spark.script.location":"s3://aws-glue-assets-896271372615-ap-south-1/scripts/test_zodiac_pg_to_redshift.py","spark.executor.memory":"10g","spark.driver.extraClassPath":"/usr/lib/livy/rsc-jars/*:/usr/lib/livy/repl_2.12-jars/*:/usr/lib/hadoop-lzo/lib/*:/usr/lib/hadoop/hadoop-aws.jar:/usr/share/aws/aws-java-sdk/*:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/*:/usr/share/aws/emr/emrfs/auxlib/*:/usr/share/aws/emr/goodies/lib/emr-spark-goodies.jar:/usr/share/aws/emr/goodies/lib/emr-serverless-spark-goodies.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/*:/usr/share/aws/hmclient/lib/aws-glue-datacatalog-spark-client.jar:/usr/share/java/Hive-JSON-Serde/hive-openx-serde.jar:/usr/share/aws/sagemaker-spark-sdk/lib/sagemaker-spark-sdk.jar:/usr/share/aws/emr/s3select/lib/emr-s3-select-spark-connector.jar:/docker/usr/lib/hadoop-lzo/lib/*:/docker/usr/lib/hadoop/hadoop-aws.jar:/docker/usr/share/aws/aws-java-sdk/*:/docker/usr/share/aws/emr/emrfs/conf:/docker/usr/share/aws/emr/emrfs/lib/*:/docker/usr/share/aws/emr/emrfs/auxlib/*:/docker/usr/share/aws/emr/goodies/lib/emr-spark-goodies.jar:/docker/usr/share/aws/emr/security/conf:/docker/usr/share/aws/emr/security/lib/*:/docker/usr/share/aws/hmclient/lib/aws-glue-datacatalog-spark-client.jar:/docker/usr/share/java/Hive-JSON-Serde/hive-openx-serde.jar:/docker/usr/share/aws/sagemaker-spark-sdk/lib/sagemaker-spark-sdk.jar:/docker/usr/share/aws/emr/s3select/lib/emr-s3-select-spark-connector.jar:/usr/share/aws/redshift/jdbc/RedshiftJDBC.jar:/usr/share/aws/redshift/spark-redshift/lib/*:/usr/share/aws/iceberg/lib/iceberg-emr-common.jar:/usr/share/aws/iceberg/lib/iceberg-spark3-runtime.jar:/usr/share/aws/delta/lib/delta-spark.jar:/usr/share/aws/delta/lib/delta-storage.jar:/usr/lib/hudi/hudi-spark-bundle.jar","spark.eventLog.dir":"file:///var/log/spark/apps","spark.dynamicAllocation.enabled":"false","spark.executor.extraClassPath":"/usr/lib/hadoop-lzo/lib/*:/usr/lib/hadoop/hadoop-aws.jar:/usr/share/aws/aws-java-sdk/*:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/*:/usr/share/aws/emr/emrfs/auxlib/*:/usr/share/aws/emr/goodies/lib/emr-spark-goodies.jar:/usr/share/aws/emr/goodies/lib/emr-serverless-spark-goodies.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/*:/usr/share/aws/hmclient/lib/aws-glue-datacatalog-spark-client.jar:/usr/share/java/Hive-JSON-Serde/hive-openx-serde.jar:/usr/share/aws/sagemaker-spark-sdk/lib/sagemaker-spark-sdk.jar:/usr/share/aws/emr/s3select/lib/emr-s3-select-spark-connector.jar:/docker/usr/lib/hadoop-lzo/lib/*:/docker/usr/lib/hadoop/hadoop-aws.jar:/docker/usr/share/aws/aws-java-sdk/*:/docker/usr/share/aws/emr/emrfs/conf:/docker/usr/share/aws/emr/emrfs/lib/*:/docker/usr/share/aws/emr/emrfs/auxlib/*:/docker/usr/share/aws/emr/goodies/lib/emr-spark-goodies.jar:/docker/usr/share/aws/emr/security/conf:/docker/usr/share/aws/emr/security/lib/*:/docker/usr/share/aws/hmclient/lib/aws-glue-datacatalog-spark-client.jar:/docker/usr/share/java/Hive-JSON-Serde/hive-openx-serde.jar:/docker/usr/share/aws/sagemaker-spark-sdk/lib/sagemaker-spark-sdk.jar:/docker/usr/share/aws/emr/s3select/lib/emr-s3-select-spark-connector.jar:/usr/share/aws/redshift/jdbc/RedshiftJDBC.jar:/usr/share/aws/redshift/spark-redshift/lib/*:/usr/share/aws/iceberg/lib/iceberg-emr-common.jar:/usr/share/aws/iceberg/lib/iceberg-spark3-runtime.jar:/usr/share/aws/delta/lib/delta-spark.jar:/usr/share/aws/delta/lib/delta-storage.jar:/usr/lib/hudi/hudi-spark-bundle.jar","spark.sql.catalogImplementation":"hive","spark.executor.cores":"4","spark.history.ui.port":"18080","spark.python.dependencyVersion.wrapt":"1.16.0","spark.metrics.conf.*.source.s3.class":"org.apache.spark.metrics.source.S3FileSystemSource","spark.hadoop.fs.s3.customAWSCredentialsProvider":"com.amazonaws.auth.DefaultAWSCredentialsProviderChain","spark.glue.JOB_NAME":"test_zodiac_pg_to_redshift","spark.hadoop.fs.AbstractFileSystem.s3.impl":"org.apache.hadoop.fs.s3.EMRFSDelegate","spark.python.dependencyVersion.yarl":"1.9.4","spark.python.dependencyVersion.tenacity":"9.0.0","spark.metrics.conf.*.sink.GlueCloudwatch.namespace":"Glue","spark.glue.nativeconnectors":"","spark.dynamicAllocation.minExecutors":"1","spark.python.dependencyVersion.tzdata":"2024.1","spark.dynamicAllocation.initialExecutors":"3","spark.files.useFetchCache":"false","spark.blacklist.decommissioning.enabled":"true","spark.python.dependencyVersion.requests":"2.32.2","spark.metrics.conf.*.sink.GlueCloudwatch.glueVersion":"5.0","spark.decommissioning.timeout.threshold":"20","spark.metrics.conf.*.sink.GlueCloudwatch.class":"org.apache.spark.metrics.sink.GlueCloudwatchSink","spark.python.dependencyVersion.six":"1.16.0","spark.python.dependencyVersion.frozenlist":"1.4.1","spark.emr-serverless.client.describe.batch.size":"100","spark.metrics.conf.*.sink.GlueCloudwatch.legacyMetrics":"true","spark.python.dependencyVersion.s3transfer":"0.10.2","spark.hadoop.mapreduce.fileoutputcommitter.cleanup-failures.ignored.emr_internal_use_only.EmrFileSystem":"true","spark.glue.additionalParams.PROXY_DISABLED_V2":"false","spark.app.id":"spark-application-1761565571107","spark.python.dependencyVersion.amzn-awsgluemlpythonlibs":"1.0","spark.glue.jobLanguage":"python","spark.hadoop.mapred.output.direct.NativeS3FileSystem":"true","spark.sql.shuffle.partitions":"36"},"Hadoop Properties":{"hadoop.service.shutdown.timeout":"30s","yarn.resourcemanager.amlauncher.thread-count":"50","yarn.sharedcache.enabled":"false","fs.s3a.connection.maximum":"500","yarn.nodemanager.numa-awareness.numactl.cmd":"/usr/bin/numactl","fs.viewfs.overload.scheme.target.o3fs.impl":"org.apache.hadoop.fs.ozone.OzoneFileSystem","fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","yarn.timeline-service.timeline-client.number-of-async-entities-to-merge":"10","hadoop.security.kms.client.timeout":"60","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.jobhistory.loadedjob.tasks.max":"-1","yarn.resourcemanager.application-tag-based-placement.enable":"false","mapreduce.framework.name":"yarn","yarn.sharedcache.uploader.server.thread-count":"50","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds.min":"3600","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"60","yarn.nodemanager.node-attributes.resync-interval-ms":"120000","yarn.nodemanager.container-log-monitor.interval-ms":"60000","yarn.resourcemanager.decommissioning-nodes-watcher.wait-for-shuffle-data":"false","hadoop.security.groups.cache.background.reload.threads":"3","yarn.resourcemanager.webapp.cross-origin.enabled":"false","fs.AbstractFileSystem.ftp.impl":"org.apache.hadoop.fs.ftp.FtpFs","fs.viewfs.overload.scheme.target.gs.impl":"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","hadoop.registry.secure":"false","hadoop.shell.safely.delete.limit.num.files":"100","fs.s3.buffer.dir":"/tmp/hadoop/s3","mapreduce.job.acl-view-job":" ","fs.s3a.retry.limit":"7","mapreduce.jobhistory.loadedjobs.cache.size":"5","yarn.timeline-service.client.create-entity-file-path":"false","mapreduce.outputcommitter.factory.scheme.abfs":"org.apache.hadoop.fs.azurebfs.commit.AzureManifestCommitterFactory","yarn.router.interceptor.user-thread-pool.allow-core-thread-time-out":"false","yarn.log-aggregation.enable-local-cleanup":"true","fs.viewfs.overload.scheme.target.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","dfs.namenode.handler.count":"64","yarn.nodemanager.amrmproxy.enabled":"false","yarn.timeline-service.entity-group-fs-store.with-user-dir":"false","parquet.enable.summary-metadata":"false","mapreduce.shuffle.pathcache.expire-after-access-minutes":"5","mapreduce.input.fileinputformat.split.minsize":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","dfs.namenode.replication.max-streams":"20","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","mapreduce.tasktracker.http.threads":"60","yarn.nodemanager.runtime.linux.docker.allowed-container-runtimes":"runc","fs.viewfs.overload.scheme.target.http.impl":"org.apache.hadoop.fs.http.HttpFileSystem","fs.s3.getObject.initialSocketTimeoutMilliseconds":"2000","yarn.nodemanager.least-load-policy-selector.fail-on-error":"true","yarn.resourcemanager.nodemanagers.heartbeat-interval-slowdown-factor":"1.0","yarn.sharedcache.checksum.algo.impl":"org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl","yarn.router.submit.interval.time":"10ms","yarn.nodemanager.amrmproxy.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor","dfs.datanode.data.dir":"file:///mnt/hdfs","dfs.replication":"2","yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size":"10485760","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","mapreduce.task.profile.maps":"0-2","yarn.scheduler.include-port-in-node-name":"false","yarn.resourcemanager.scheduler.autocorrect.container.allocation":"false","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","yarn.resourcemanager.node-removal-untracked.timeout-ms":"120000","yarn.router.interceptor.user-thread-pool.keep-alive-time":"30s","mapreduce.am.max-attempts":"2","hadoop.security.kms.client.failover.sleep.base.millis":"100","fs.s3a.connection.ttl":"5m","yarn.router.asc-interceptor-max-size":"1MB","mapreduce.jobhistory.webapp.https.address":"0.0.0.0:19890","yarn.node-labels.fs-store.impl.class":"org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore","yarn.nodemanager.collector-service.address":"${yarn.nodemanager.hostname}:8048","fs.trash.checkpoint.interval":"0","yarn.nodemanager.opportunistic-containers-queue-policy":"BY_QUEUE_LEN","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","yarn.federation.gpg.policy.generator.load-based.pending.maximum":"1000","yarn.resourcemanager.node-ip-cache.expiry-interval-secs":"-1","hadoop.http.authentication.signature.secret.file":"*********(redacted)","fs.s3a.committer.magic.track.commits.in.memory.enabled":"true","hadoop.jetty.logs.serve.aliases":"true","yarn.resourcemanager.placement-constraints.handler":"disabled","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"1000","yarn.nodemanager.aux-services.manifest.enabled":"false","dfs.hosts.exclude":"/emr/instance-controller/lib/dfs.hosts.exclude","yarn.resourcemanager.system-metrics-publisher.enabled":"false","yarn.resourcemanager.placement-constraints.algorithm.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm","yarn.sharedcache.webapp.address":"0.0.0.0:8788","fs.s3a.select.input.csv.quote.escape.character":"\\\\","yarn.resourcemanager.delegation.token.renew-interval":"*********(redacted)","yarn.sharedcache.nm.uploader.replication.factor":"10","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","mapreduce.reduce.skip.proc-count.auto-incr":"true","yarn.dispatcher.print-thread-pool.core-pool-size":"1","hadoop.security.group.mapping.ldap.posix.attr.gid.name":"gidNumber","ipc.client.fallback-to-simple-auth-allowed":"false","yarn.nodemanager.resource.memory.enforced":"true","yarn.federation.gpg.policy.generator.interval-ms":"3600000","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.enable-batch":"false","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.federation.state-store.sql.idle-time-out":"10m","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","io.compression.codec.zstd.level":"3","yarn.nodemanager.runtime.linux.runc.host-pid-namespace.allowed":"false","hadoop.metrics.jvm.use-thread-mxbean":"false","ipc.[port_number].faircallqueue.multiplexer.weights":"8,4,2,1","yarn.acl.reservation-enable":"false","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","ipc.[port_number].callqueue.overflow.trigger.failover":"false","yarn.app.mapreduce.am.hard-kill-timeout-ms":"10000","fs.s3a.etag.checksum.enabled":"false","yarn.nodemanager.container-metrics.enable":"true","mapreduce.outputcommitter.factory.scheme.gs":"org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterFactory","ha.health-monitor.rpc.connect.max.retries":"1","yarn.timeline-service.client.fd-clean-interval-secs":"60","yarn.resourcemanager.nodemanagers.heartbeat-interval-scaling-enable":"false","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","hadoop.common.configuration.version":"3.0.0","yarn.nodemanager.remote-app-log-dir-suffix":"logs","yarn.nodemanager.container-log-monitor.dir-size-limit-bytes":"1000000000","yarn.nodemanager.windows-container.cpu-limit.enabled":"false","yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed":"false","file.blocksize":"67108864","hadoop.http.idle_timeout.ms":"60000","hadoop.registry.zk.retry.ceiling.ms":"60000","ipc.client.connection.idle-scan-interval.ms":"10000","yarn.scheduler.configuration.leveldb-store.path":"${hadoop.tmp.dir}/yarn/system/confstore","yarn.sharedcache.store.in-memory.initial-delay-mins":"10","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","mapreduce.map.skip.proc-count.auto-incr":"true","fs.s3a.committer.name":"magicv2","mapreduce.task.profile.reduces":"0-2","hadoop.zk.num-retries":"1000","yarn.webapp.xfs-filter.enabled":"true","fs.viewfs.overload.scheme.target.hdfs.impl":"org.apache.hadoop.hdfs.DistributedFileSystem","seq.io.sort.mb":"100","fs.iostatistics.logging.level":"debug","yarn.scheduler.configuration.max.version":"100","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"${yarn.resourcemanager.hostname}:8030","yarn.node-labels.enabled":"false","yarn.resourcemanager.webapp.ui-actions.enabled":"true","mapreduce.task.timeout":"600000","yarn.sharedcache.client-server.thread-count":"50","hadoop.security.groups.shell.command.timeout":"0s","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.nodemanager.elastic-memory-control.oom-handler":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.DefaultOOMHandler","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"file:///","yarn.minicluster.use-rpc":"false","fs.s3.impl":"com.amazon.ws.emr.hadoop.fs.EmrFileSystem","ipc.[port_number].decay-scheduler.decay-factor":"0.5","fs.har.impl.disable.cache":"true","yarn.webapp.ui2.enable":"false","io.compression.codec.bzip2.library":"system-native","yarn.webapp.filter-invalid-xml-chars":"false","fs.s3.buckets.create.region":"us-east-1","yarn.nodemanager.runtime.linux.runc.layer-mounts-interval-secs":"600","fs.s3a.select.input.csv.record.delimiter":"\\n","fs.s3a.change.detection.source":"etag","ipc.[port_number].backoff.enable":"false","yarn.nodemanager.distributed-scheduling.enabled":"false","yarn.federation.cache.class":"org.apache.hadoop.yarn.server.federation.cache.FederationJCache","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"${yarn.resourcemanager.hostname}:8090","yarn.webapp.enable-rest-app-submissions":"true","mapreduce.jobhistory.address":"0.0.0.0:10020","yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.is.minicluster":"false","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","hadoop.proxyuser.livy.groups":"*","fs.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A","ipc.server.metrics.update.runner.interval":"5000","mapreduce.task.combine.progress.records":"10000","yarn.resourcemanager.epoch.range":"0","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.runtime.linux.runc.image-toplevel-dir":"/runc-root","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","io.compression.codec.lz4.use.lz4hc":"false","fs.AbstractFileSystem.wasbs.impl":"org.apache.hadoop.fs.azure.Wasbs","yarn.timeline-service.entity-group-fs-store.cache-store-class":"org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore","yarn.nodemanager.runtime.linux.runc.allowed-container-networks":"host,none,bridge","fs.ftp.transfer.mode":"BLOCK_TRANSFER_MODE","ipc.server.log.slow.rpc":"false","ipc.server.reuseaddr":"true","fs.ftp.timeout":"0","yarn.resourcemanager.node-labels.provider.fetch-interval-ms":"1800000","fs.AbstractFileSystem.o3fs.impl":"org.apache.hadoop.fs.ozone.OzFs","yarn.router.webapp.https.address":"0.0.0.0:8091","yarn.resourcemanager.enable-node-untracked-without-include-path":"true","yarn.nodemanager.webapp.cross-origin.enabled":"false","yarn.federation.gpg.subcluster.heartbeat.expiration-ms":"30m","fs.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","yarn.resourcemanager.auto-update.containers":"false","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","yarn.scheduler.configuration.zk-store.parent-path":"/confstore","yarn.nodemanager.default-container-executor.log-dirs.permissions":"710","yarn.app.attempt.diagnostics.limit.kc":"64","fs.viewfs.overload.scheme.target.swebhdfs.impl":"org.apache.hadoop.hdfs.web.SWebHdfsFileSystem","yarn.client.failover-no-ha-proxy-provider":"org.apache.hadoop.yarn.client.DefaultNoHARMFailoverProxyProvider","fs.s3a.change.detection.mode":"server","hadoop.proxyuser.presto.hosts":"*","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"-1","io.compression.codecs":"org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.SnappyCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec","fs.AbstractFileSystem.abfs.impl":"org.apache.hadoop.fs.azurebfs.Abfs","yarn.timeline-service.writer.flush-interval-seconds":"60","fs.s3a.fast.upload.active.blocks":"4","yarn.resourcemanager.submission-preprocessor.enabled":"false","hadoop.security.credential.clear-text-fallback":"true","yarn.nodemanager.collector-service.thread-count":"5","ipc.[port_number].scheduler.impl":"org.apache.hadoop.ipc.DefaultRpcScheduler","fs.azure.secure.mode":"false","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","yarn.timeline-service.writer.async.queue.capacity":"100","yarn.router.webapp.appsinfo-cached-count":"100","yarn.resourcemanager.fs.state-store.num-retries":"0","yarn.resourcemanager.nodemanager-connect-retries":"10","yarn.nodemanager.log-aggregation.num-log-files-per-app":"30","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","fs.s3a.committer.magic.enabled":"true","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","yarn.system-metrics-publisher.enabled":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","yarn.resourcemanager.nodemanagers.heartbeat-interval-speedup-factor":"1.0","mapreduce.task.io.sort.factor":"48","yarn.nodemanager.amrmproxy.client.thread-count":"25","yarn.nodemanager.runtime.linux.docker.docker-client-credential-provider.class":"org.apache.hadoop.yarn.util.ReadConfigFileDockerClientCredentialProvider","ha.failover-controller.new-active.rpc-timeout.ms":"60000","yarn.nodemanager.container-localizer.java.opts":"-Xmx256m","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","yarn.nodemanager.windows-container.memory-limit.enabled":"false","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.minicluster.fixed.ports":"false","hadoop.tags.system":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n      ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.cluster.max-application-priority":"0","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","yarn.nodemanager.least-load-policy-selector.enabled":"false","yarn.nodemanager.least-load-policy-selector.pending-container.threshold":"10000","hadoop.caller.context.signature.max.size":"40","hadoop.proxyuser.hive.groups":"*","ipc.[port_number].decay-scheduler.backoff.responsetime.enable":"false","yarn.client.load.resource-types.from-server":"false","ha.zookeeper.session-timeout.ms":"10000","ipc.[port_number].decay-scheduler.metrics.top.user.count":"10","mapreduce.map.java.opts":"-Xmx2458m","tfile.io.chunk.size":"1048576","yarn.dispatcher.print-events-info.threshold":"5000","yarn.nodemanager.log-container-debug-info-on-error.enabled":"false","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.kms.client.failover.sleep.max.millis":"2000","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","hive.metastore.warehouse.dir":"/tmp/spark-warehouse","yarn.scheduler.configuration.store.max-logs":"1000","yarn.nodemanager.dispatcher.metric.enable":"false","yarn.nodemanager.node-attributes.provider.fetch-interval-ms":"600000","yarn.nodemanager.local-cache.max-files-per-directory":"8192","hadoop.http.cross-origin.enabled":"false","hadoop.zk.acl":"world:anyone:rwcda","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.num-manifests-to-cache":"10","mapreduce.map.sort.spill.percent":"0.80","yarn.timeline-service.entity-group-fs-store.scan-interval-seconds":"60","dfs.datanode.fsdataset.volume.choosing.policy":"org.apache.hadoop.hdfs.server.datanode.fsdataset.AvailableSpaceVolumeChoosingPolicy","yarn.node-attribute.fs-store.impl.class":"org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore","fs.s3a.retry.interval":"500ms","yarn.timeline-service.client.best-effort":"false","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","hadoop.security.group.mapping.ldap.posix.attr.uid.name":"uidNumber","yarn.timeline-service.client.fd-forced-timeout-enabled":"false","fs.AbstractFileSystem.swebhdfs.impl":"org.apache.hadoop.fs.SWebHdfs","yarn.nodemanager.elastic-memory-control.timeout-sec":"5","fs.s3a.select.enabled":"true","mapreduce.ifile.readahead":"true","dynamodb.yarn.enabled":"false","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","yarn.timeline-service.reader.webapp.address":"${yarn.timeline-service.webapp.address}","yarn.resourcemanager.placement-constraints.algorithm.pool-size":"1","yarn.timeline-service.hbase.coprocessor.jar.hdfs.location":"/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","yarn.webapp.ui1.tools.enable":"true","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler","yarn.app.mapreduce.am.command-opts":"-Xmx4915m","hadoop.http.sni.host.check.enabled":"false","mapreduce.cluster.local.dir":"/mnt/mapred,/mnt1/mapred","hadoop.proxyuser.hue.hosts":"*","io.mapfile.bloom.error.rate":"0.005","fs.client.resolve.topology.enabled":"false","hadoop.proxyuser.hue.groups":"*","yarn.nodemanager.runtime.linux.allowed-runtimes":"default","yarn.sharedcache.store.class":"org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","fs.getspaceused.jitterMillis":"60000","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","ipc.scheduler.impl":"org.apache.hadoop.ipc.DefaultRpcScheduler","yarn.resourcemanager.zk-client-ssl.enabled":"false","io.skip.checksum.errors":"false","yarn.nodemanager.log.trigger.delete.by-size.enabled":"false","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","yarn.timeline-service.webapp.xfs-filter.xframe-options":"SAMEORIGIN","fs.s3a.connection.timeout":"200s","yarn.app.mapreduce.am.webapp.https.enabled":"false","mapreduce.job.max.split.locations":"15","yarn.resourcemanager.nm-container-queuing.max-queue-length":"15","yarn.resourcemanager.delegation-token.always-cancel":"*********(redacted)","hadoop.registry.zk.session.timeout.ms":"60000","yarn.federation.cache-ttl.secs":"300","mapreduce.jvm.system-properties-to-log":"os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name","yarn.resourcemanager.opportunistic-container-allocation.nodes-used":"10","yarn.timeline-service.entity-group-fs-store.active-dir":"/tmp/entity-file-history/active","mapreduce.shuffle.transfer.buffer.size":"131072","yarn.timeline-service.client.retry-interval-ms":"1000","yarn.timeline-service.flowname.max-size":"0","yarn.http.policy":"HTTP_ONLY","fs.s3a.socket.send.buffer":"8192","fs.AbstractFileSystem.abfss.impl":"org.apache.hadoop.fs.azurebfs.Abfss","yarn.sharedcache.uploader.server.address":"0.0.0.0:8046","yarn.resourcemanager.delegation-token.max-conf-size-bytes":"*********(redacted)","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","yarn.minicluster.yarn.nodemanager.resource.memory-mb":"4096","mapreduce.job.emit-timeline-data":"false","yarn.nodemanager.resource.system-reserved-memory-mb":"-1","hadoop.kerberos.min.seconds.before.relogin":"60","mapreduce.jobhistory.move.thread-count":"3","yarn.resourcemanager.application-tag-based-placement.force-lowercase":"true","yarn.resourcemanager.admin.client.thread-count":"1","yarn.dispatcher.drain-events.timeout":"300000","ipc.[port_number].decay-scheduler.backoff.responsetime.thresholds":"10s,20s,30s,40s","fs.s3a.buffer.dir":"/mnt/s3","hadoop.ssl.enabled.protocols":"TLSv1.2","mapreduce.jobhistory.admin.address":"0.0.0.0:10033","yarn.log-aggregation-status.time-out.ms":"600000","ipc.server.max.response.size":"1048576","fs.s3a.accesspoint.required":"false","mapreduce.shuffle.port":"13562","yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory":"10","yarn.nodemanager.health-checker.interval-ms":"600000","yarn.resourcemanager.proxy.connection.timeout":"60000","yarn.router.clientrm.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor","yarn.resourcemanager.zk-appid-node.split-index":"0","ftp.blocksize":"67108864","yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions":"read","yarn.router.rmadmin.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor","yarn.nodemanager.log-container-debug-info.enabled":"true","yarn.resourcemanager.activities-manager.app-activities.max-queue-length":"100","yarn.resourcemanager.application-https.policy":"NONE","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms":"20","yarn.nodemanager.delete.debug-delay-sec":"0","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","yarn.nodemanager.container-retry-minimum-interval-ms":"1000","yarn.federation.gpg.application.cleaner.contact.router.spec":"3,10,600000","hadoop.security.groups.cache.secs":"300","yarn.federation.enabled":"false","yarn.workflow-id.tag-prefix":"workflowid:","fs.azure.local.sas.key.mode":"false","yarn.federation.gpg.policy.generator.class":"org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator.NoOpGlobalPolicy","ipc.maximum.data.length":"134217728","mapreduce.shuffle.max.threads":"0","yarn.router.pipeline.cache-max-size":"25","yarn.resourcemanager.nm-container-queuing.load-comparator":"QUEUE_LENGTH","ipc.server.tcpnodelay":"true","yarn.resourcemanager.resource-tracker.nm.ip-hostname-check":"false","hadoop.security.authorization":"false","yarn.app.mapreduce.am.jhs.backup.enabled":"true","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","yarn.nodemanager.amrmproxy.ha.enable":"false","yarn.router.webapp.appsinfo-enabled":"false","fs.AbstractFileSystem.gs.impl":"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","mapreduce.jobhistory.http.policy":"HTTP_ONLY","yarn.sharedcache.store.in-memory.check-period-mins":"720","mapreduce.reduce.java.opts":"-Xmx4916m","hive.metastore.client.factory.class":"com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory","hadoop.security.group.mapping.ldap.ssl":"false","fs.s3a.downgrade.syncable.exceptions":"true","yarn.client.application-client-protocol.poll-interval-ms":"200","yarn.scheduler.configuration.leveldb-store.compaction-interval-secs":"86400","yarn.timeline-service.writer.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl","ha.zookeeper.parent-znode":"/hadoop-ha","yarn.resourcemanager.submission-preprocessor.file-refresh-interval-ms":"60000","dfs.namenode.safemode.extension":"5000","yarn.nodemanager.log-aggregation.policy.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","yarn.resourcemanager.placement-constraints.scheduler.pool-size":"1","yarn.resourcemanager.activities-manager.cleanup-interval-ms":"5000","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","dfs.namenode.replication.max-streams-hard-limit":"40","ipc.[port_number].identity-provider.impl":"org.apache.hadoop.ipc.UserIdentityProvider","yarn.nodemanager.recovery.supervised":"false","yarn.sharedcache.admin.thread-count":"1","yarn.resourcemanager.ha.automatic-failover.enabled":"true","yarn.nodemanager.container-log-monitor.total-size-limit-bytes":"10000000000","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.federation.amrmproxy.register.uam.retry-count":"3","yarn.nodemanager.health-checker.scripts":"script","yarn.resourcemanager.address":"${yarn.resourcemanager.hostname}:8032","ipc.client.ping":"true","mapreduce.task.local-fs.write-limit.bytes":"-1","mapred.output.committer.class":"org.apache.hadoop.mapred.DirectFileOutputCommitter","fs.adl.oauth2.access.token.provider.type":"*********(redacted)","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","yarn.nodemanager.resource-plugins.gpu.docker-plugin":"nvidia-docker-v1","fs.s3a.multipart.purge":"false","yarn.scheduler.configuration.store.class":"file","yarn.resourcemanager.nm-container-queuing.queue-limit-stdev":"1.0f","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","ipc.client.bind.wildcard.addr":"false","yarn.resourcemanager.webapp.rest-csrf.enabled":"false","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","yarn.federation.cache-entity.nums":"1000","hadoop.security.resolver.impl":"org.apache.hadoop.net.DNSDomainNameResolver","hive.metastore.connect.retries":"15","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"96","yarn.nodemanager.runtime.linux.docker.image-update":"false","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","fs.viewfs.overload.scheme.target.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","yarn.dispatcher.cpu-monitor.samples-per-min":"60","hadoop.security.token.service.use_ip":"*********(redacted)","yarn.nodemanager.runtime.linux.docker.allowed-container-networks":"host,none,bridge","yarn.nodemanager.node-labels.resync-interval-ms":"120000","hadoop.tmp.dir":"/tmp/hadoop","mapreduce.job.maps":"16","mapreduce.jobhistory.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","yarn.nodemanager.containers-launcher.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher","yarn.resourcemanager.decommissioning-nodes-watcher.wait-for-applications":"false","yarn.rm.system-metrics-publisher.emit-container-events":"false","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","fs.AbstractFileSystem.wasb.impl":"org.apache.hadoop.fs.azure.Wasb","mapreduce.client.submit.file.replication":"10","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"128M","yarn.resourcemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","ipc.callqueue.impl":"java.util.concurrent.LinkedBlockingQueue","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","ipc.server.purge.interval":"15","dfs.namenode.name.dir":"file:///mnt/namenode","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold":"1","yarn.nodemanager.webapp.rest-csrf.enabled":"false","fs.ftp.host.port":"21","ipc.ping.interval":"60000","yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size":"10","yarn.router.interceptor.user-thread-pool.maximum-pool-size":"5","yarn.resourcemanager.admin.address":"${yarn.resourcemanager.hostname}:8033","file.client-write-packet-size":"65536","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"true","hadoop.security.key.default.bitlength":"256","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","yarn.nodemanager.log.deletion-threads-count":"4","fs.s3a.committer.abort.pending.uploads":"true","yarn.webapp.filter-entity-list-by-user":"false","yarn.resourcemanager.activities-manager.app-activities.ttl-ms":"600000","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"200","yarn.nodemanager.localizer.client.thread-count":"5","yarn.federation.gpg.policy.generator.load-based.edit.maximum":"3","io.erasurecode.codec.rs.rawcoders":"rs_native,rs_java","io.erasurecode.codec.rs-legacy.rawcoders":"rs-legacy_java","yarn.sharedcache.admin.address":"0.0.0.0:8047","yarn.resourcemanager.placement-constraints.algorithm.iterator":"SERIAL","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec","mapreduce.job.cache.limit.max-resources-mb":"0","fs.s3a.connection.ssl.enabled":"true","yarn.nodemanager.process-kill-wait.ms":"5000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","yarn.apps.cache.size":"1000","yarn.app.mapreduce.am.webapp.https.client.auth":"false","hadoop.workaround.non.threadsafe.getpwuid":"true","fs.df.interval":"60000","ipc.[port_number].decay-scheduler.thresholds":"13,25,50","yarn.federation.gpg.webapp.https.address":"0.0.0.0:8070","ipc.server.read.threadpool.size":"1","fs.s3a.audit.enabled":"true","fs.s3a.multiobjectdelete.enable":"true","yarn.sharedcache.cleaner.resource-sleep-ms":"0","ipc.server.read.connection-queue.size":"100","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","hadoop.shell.missing.defaultFs.warning":"false","io.file.buffer.size":"65536","fs.viewfs.overload.scheme.target.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","dfs.permissions.superusergroup":"hadoop","hadoop.security.group.mapping.ldap.search.attr.member":"member","yarn.nodemanager.shuffledata-monitor.interval-ms":"30000","hadoop.security.random.device.file.path":"/dev/urandom","ipc.cost-provider.impl":"org.apache.hadoop.ipc.DefaultCostProvider","hadoop.security.sensitive-config-keys":"*********(redacted)","fs.viewfs.overload.scheme.target.file.impl":"org.apache.hadoop.fs.LocalFileSystem","yarn.federation.gpg.application.cleaner.interval-ms":"-1s","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","yarn.intermediate-data-encryption.enable":"false","yarn.nodemanager.least-load-policy-selector.use-active-core":"false","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.nodemanager.container.stderr.pattern":"{*stderr*,*STDERR*}","fs.s3bfs.impl":"org.apache.hadoop.fs.s3.S3FileSystem","yarn.apps.cache.expire":"30s","ipc.server.log.slow.rpc.threshold.ms":"0","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/tmp/hadoop-yarn/staging","mapreduce.reduce.shuffle.read.timeout":"180000","fs.s3.customAWSCredentialsProvider":"com.amazonaws.auth.DefaultAWSCredentialsProviderChain","hadoop.http.cross-origin.max-age":"1800","io.erasurecode.codec.xor.rawcoders":"xor_native,xor_java","fs.s3a.connection.establish.timeout":"30s","mapreduce.job.running.map.limit":"0","yarn.minicluster.control-resource-monitoring":"false","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","yarn.apps.cache.enable":"false","yarn.federation.non-ha.enabled":"false","yarn.federation.state-store.class":"org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore","yarn.federation.state-store.heartbeat.initial-delay":"30s","mapreduce.reduce.log.level":"INFO","hadoop.security.dns.log-slow-lookups.threshold.ms":"1000","mapreduce.job.ubertask.enable":"false","adl.http.timeout":"-1","yarn.resourcemanager.placement-constraints.retry-attempts":"3","hadoop.caller.context.enabled":"false","hadoop.security.group.mapping.ldap.num.attempts":"3","yarn.nodemanager.vmem-pmem-ratio":"2.1","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","hadoop.zk.timeout-ms":"10000","yarn.nodemanager.resource.pcores-vcores-multiplier":"1.0","yarn.nodemanager.runtime.linux.sandbox-mode":"disabled","ipc.client.connect.max.retries.on.sasl":"5","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.viewfs.overload.scheme.target.webhdfs.impl":"org.apache.hadoop.hdfs.web.WebHdfsFileSystem","yarn.router.webapp.proxy.enable":"true","fs.s3a.committer.threads":"8","hadoop.zk.retry-interval-ms":"1000","hadoop.http.metrics.enabled":"true","hadoop.security.crypto.buffer.size":"8192","yarn.nodemanager.node-labels.provider.fetch-interval-ms":"600000","mapreduce.jobhistory.recovery.store.leveldb.path":"${hadoop.tmp.dir}/mapred/history/recoverystore","yarn.client.failover-retries-on-socket-timeouts":"0","fs.s3a.ssl.channel.mode":"default_jsse","yarn.nodemanager.resource.memory.enabled":"false","fs.azure.authorization.caching.enable":"true","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","mapreduce.job.finish-when-all-reducers-done":"true","hadoop.registry.jaas.context":"Client","yarn.resourcemanager.delegation.token.remove-scan-interval":"*********(redacted)","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","io.map.index.interval":"128","yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms":"100","fs.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","mapreduce.job.counters.max":"120","ipc.identity-provider.impl":"org.apache.hadoop.ipc.UserIdentityProvider","mapreduce.jobhistory.webapp.rest-csrf.enabled":"false","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","yarn.federation.amrmproxy.register.uam.interval":"100ms","yarn.resourcemanager.node-labels.provider.update-newly-registered-nodes-interval-ms":"30000","fs.s3a.change.detection.version.required":"true","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","yarn.timeline-service.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/timeline","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","hadoop.security.auth_to_local.mechanism":"hadoop","yarn.timeline-service.app-collector.linger-period.ms":"60000","yarn.fs-store.file.replication":"0","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.reservation-system.planfollower.time-step":"1000","yarn.resourcemanager.proxy.timeout.enabled":"true","yarn.resourcemanager.activities-manager.scheduler-activities.ttl-ms":"600000","yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed":"true","hadoop.proxyuser.hadoop.groups":"*","yarn.webapp.api-service.enable":"false","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","fs.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","yarn.nodemanager.container.stderr.tail.bytes":"4096","yarn.nodemanager.disk-health-checker.disk-free-space-threshold.enabled":"true","io.compression.codec.snappy.buffersize":"262144","hadoop.security.group.mapping.ldap.read.timeout.ms":"60000","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","mapreduce.outputcommitter.factory.scheme.s3a":"org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory","io.erasurecode.codec.native.enabled":"true","hadoop.security.groups.cache.background.reload":"false","yarn.nodemanager.container-monitor.enabled":"true","yarn.nodemanager.elastic-memory-control.enabled":"false","fs.AbstractFileSystem.ofs.impl":"org.apache.hadoop.fs.ozone.RootedOzFs","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","yarn.resourcemanager.rm.container-allocation.expiry-interval-ms":"600000","yarn.nodemanager.container-executor.exit-code-file.timeout-ms":"2000","mapreduce.fileoutputcommitter.algorithm.version":"2","yarn.router.webapp.cross-origin.enabled":"false","yarn.resourcemanager.work-preserving-recovery.enabled":"true","mapreduce.map.skip.maxrecords":"0","yarn.sharedcache.root-dir":"/sharedcache","fs.s3a.retry.throttle.limit":"20","yarn.nodemanager.master-hostname-resolution-check.interval-ms":"12000","fs.trash.clean.trashroot.enable":"false","hadoop.http.authentication.type":"simple","fs.viewfs.overload.scheme.target.oss.impl":"org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem","mapreduce.job.jvm.numtasks":"20","yarn.federation.gpg.policy.generator.interval":"1h","mapreduce.job.cache.limit.max-resources":"0","mapreduce.fileoutputcommitter.marksuccessfuljobs":"false","mapreduce.task.userlog.limit.kb":"0","ipc.[port_number].weighted-cost.handler":"1","yarn.resourcemanager.scheduler.monitor.enable":"false","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","yarn.nodemanager.resource-monitor.interval-ms":"3000","yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices":"auto","mapreduce.job.sharedcache.mode":"disabled","yarn.app.mapreduce.am.jhs.backup-dir":"file:///var/log/hadoop-mapreduce/history","yarn.federation.state-store.sql.minimum-idle":"1","yarn.nodemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.shuffle.listen.queue.size":"128","yarn.scheduler.configuration.mutation.acl-policy.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy","yarn.federation.gpg.subcluster.cleaner.interval-ms":"-1ms","mapreduce.map.cpu.vcores":"1","yarn.log-aggregation.file-formats":"TFile","yarn.timeline-service.client.fd-retain-secs":"300","fs.s3a.select.output.csv.field.delimiter":",","yarn.nodemanager.health-checker.timeout-ms":"1200000","hadoop.user.group.static.mapping.overrides":"dr.who=;","fs.azure.sas.expiry.period":"90d","fs.s3a.select.output.csv.record.delimiter":"\\n","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","fs.viewfs.overload.scheme.target.https.impl":"org.apache.hadoop.fs.http.HttpsFileSystem","yarn.federation.gpg.policy.generator.readonly":"false","yarn.router.deregister.subcluster.enabled":"true","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"false","yarn.router.webapp.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST","yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage":"90.0","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","yarn.nodemanager.disk-validator":"basic","yarn.app.mapreduce.client.job.max-retries":"3","fs.viewfs.overload.scheme.target.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","hadoop.security.group.mapping.ldap.connection.timeout.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","yarn.nodemanager.container-diagnostics-maximum-size":"10000","hadoop.registry.zk.retry.interval.ms":"1000","yarn.federation.gpg.scheduled.executor.threads":"10","yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms":"1000","yarn.nodemanager.aux-services.%s.classpath":"NONE","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.federation.gpg.webapp.connect-timeout":"30s","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","yarn.nodemanager.runtime.linux.runc.allowed-container-runtimes":"runc","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","fs.s3a.assumed.role.session.duration":"30m","hadoop.security.group.mapping.ldap.conversion.rule":"none","hadoop.proxyuser.livy.hosts":"*","hadoop.ssl.server.conf":"ssl-server.xml","fs.s3a.retry.throttle.interval":"100ms","yarn.router.subcluster.cleaner.interval.time":"60s","yarn.nodemanager.log.delete.threshold":"100g","seq.io.sort.factor":"100","fs.viewfs.overload.scheme.target.ofs.impl":"org.apache.hadoop.fs.ozone.RootedOzoneFileSystem","yarn.sharedcache.cleaner.initial-delay-mins":"10","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.resourcemanager.nodestore-rootdir.retry-interval-ms":"1000","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.nodemanager.runtime.linux.docker.capabilities":"CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE","yarn.acl.enable":"false","yarn.timeline-service.entity-group-fs-store.done-dir":"/tmp/entity-file-history/done/","mapreduce.tasktracker.map.tasks.maximum":"1","hadoop.security.group.mapping.ldap.num.attempts.before.failover":"3","mapreduce.task.profile":"false","yarn.federation.gpg.application.cleaner.class":"org.apache.hadoop.yarn.server.globalpolicygenerator.applicationcleaner.DefaultApplicationCleaner","hadoop.prometheus.endpoint.enabled":"false","dfs.webhdfs.enabled":"true","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","mapreduce.jobhistory.always-scan-user-dir":"false","yarn.nodemanager.opportunistic-containers-use-pause-for-preemption":"false","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"nobody","mapred.output.direct.EmrFileSystem":"true","yarn.timeline-service.reader.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold":"1","yarn.resourcemanager.configuration.file-system-based-store":"/yarn/conf","lakeformation.credentials.url":"http://localhost:9998/lakeformationcredentials","fs.creation.parallel.count":"64","mapreduce.job.cache.limit.max-single-resource-mb":"0","yarn.nodemanager.runtime.linux.docker.stop.grace-period":"10","yarn.federation.state-store.sql.max-life-time":"30m","yarn.resourcemanager.resource-profiles.source-file":"resource-profiles.json","mapreduce.job.dfs.storage.capacity.kill-limit-exceed":"false","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"${yarn.log.dir}/userlogs","hadoop.security.auth_to_local":"\n      RULE:[1:$1@$0](.*@)s/@.*///L\n      RULE:[2:$1@$0](.*@)s/@.*///L\n      DEFAULT\n    ","yarn.resourcemanager.opportunistic.max.container-allocation.per.am.heartbeat":"-1","fs.automatic.close":"true","yarn.resourcemanager.delegation-token-renewer.thread-retry-interval":"*********(redacted)","yarn.resourcemanager.node-labels.am.allow-non-exclusive-allocation":"false","fs.s3a.select.input.csv.quote.character":"\"","yarn.nodemanager.hostname":"0.0.0.0","ipc.[port_number].cost-provider.impl":"org.apache.hadoop.ipc.DefaultCostProvider","yarn.nodemanager.runtime.linux.runc.manifest-to-resources-plugin":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.HdfsManifestToResourcesPlugin","yarn.nodemanager.remote-app-log-dir-include-older":"true","yarn.nodemanager.resource.memory.cgroups.swappiness":"0","fs.s3n.impl":"com.amazon.ws.emr.hadoop.fs.EmrFileSystem","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","yarn.nodemanager.runtime.linux.runc.layer-mounts-to-keep":"100","yarn.timeline-service.app-aggregation-interval-secs":"15","hadoop.security.group.mapping.ldap.search.filter.user":"(&(objectClass=user)(sAMAccountName={0}))","yarn.resourcemanager.nodestore-rootdir.num-retries":"1000","ipc.[port_number].weighted-cost.lockshared":"10","yarn.nodemanager.container-localizer.log.level":"INFO","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","dfs.namenode.replication.work.multiplier.per.iteration":"10","mapreduce.job.ubertask.maxmaps":"9","fs.s3.consistent.metadata.etag.verification.enabled":"false","fs.s3a.threads.keepalivetime":"60s","dfs.datanode.available-space-volume-choosing-policy.balanced-space-preference-fraction":"1.0","mapreduce.jobhistory.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","fs.s3a.select.output.csv.quote.escape.character":"\\\\","yarn.dispatcher.print-thread-pool.maximum-pool-size":"5","yarn.resourcemanager.delegation.token.max-lifetime":"*********(redacted)","hadoop.kerberos.keytab.login.autorenewal.enabled":"false","yarn.timeline-service.client.drain-entities.timeout.ms":"2000","hadoop.caller.context.separator":",","yarn.nodemanager.resource-plugins.fpga.vendor-plugin.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin","yarn.resourcemanager.nodemanagers.heartbeat-interval-min-ms":"1000","yarn.timeline-service.entity-group-fs-store.summary-store":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.reduce.cpu.vcores":"1","hadoop.proxyuser.oozie.groups":"*","mapreduce.job.encrypted-intermediate-data.buffer.kb":"128","hadoop.security.crypto.codec.classes.sm4.ctr.nopadding":"org.apache.hadoop.crypto.OpensslSm4CtrCryptoCodec, org.apache.hadoop.crypto.JceSm4CtrCryptoCodec","yarn.federation.gpg.webapp.read-timeout":"30s","fs.client.resolve.remote.symlinks":"true","yarn.nodemanager.webapp.https.address":"0.0.0.0:8044","hadoop.http.cross-origin.allowed-origins":"*","mapreduce.job.encrypted-intermediate-data":"false","yarn.nodemanager.disk-health-checker.disk-utilization-threshold.enabled":"true","fs.s3a.executor.capacity":"16","yarn.timeline-service.entity-group-fs-store.retain-seconds":"604800","yarn.resourcemanager.metrics.runtime.buckets":"60,300,1440","yarn.timeline-service.generic-application-history.max-applications":"10000","mapreduce.tasktracker.reduce.tasks.maximum":"1","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","yarn.node-labels.configuration-type":"centralized","fs.s3a.path.style.access":"false","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","yarn.sharedcache.store.in-memory.staleness-period-mins":"10080","fs.adl.impl":"org.apache.hadoop.fs.adl.AdlFileSystem","yarn.resourcemanager.application.max-tags":"10","hadoop.domainname.resolver.impl":"org.apache.hadoop.net.DNSDomainNameResolver","yarn.resourcemanager.nodemanager.minimum.version":"NONE","mapreduce.jobhistory.webapp.xfs-filter.xframe-options":"SAMEORIGIN","yarn.app.mapreduce.am.staging-dir.erasurecoding.enabled":"false","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","fs.AbstractFileSystem.s3.impl":"org.apache.hadoop.fs.s3.EMRFSDelegate","yarn.timeline-service.reader.webapp.https.address":"${yarn.timeline-service.webapp.https.address}","fs.ftp.data.connection.mode":"ACTIVE_LOCAL_DATA_CONNECTION_MODE","mapreduce.job.userlog.retain.hours":"48","mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed":"true","fs.azure.buffer.dir":"${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/abfs","yarn.scheduler.maximum-allocation-vcores":"4","hadoop.http.cross-origin.allowed-headers":"X-Requested-With,Content-Type,Accept,Origin","yarn.nodemanager.log-aggregation.compression-type":"none","yarn.timeline-service.version":"1.0f","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","mapreduce.reduce.maxattempts":"4","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.batch-size":"1000","hadoop.security.dns.log-slow-lookups.enabled":"false","mapreduce.job.committer.setup.cleanup.needed":"true","hadoop.security.secure.random.impl":"org.apache.hadoop.crypto.random.OpensslSecureRandom","mapreduce.job.running.reduce.limit":"0","fs.s3a.select.errors.include.sql":"false","ipc.maximum.response.length":"134217728","yarn.resourcemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","hadoop.caller.context.max.size":"128","yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed":"false","yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed":"false","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","yarn.federation.gpg.policy.generator.load-based.weight.minimum":"0","fs.s3a.fast.upload.buffer":"disk","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","yarn.log-aggregation.debug.filesize":"104857600","yarn.dispatcher.print-thread-pool.keep-alive-time":"10s","yarn.router.subcluster.heartbeat.expiration.time":"30m","fs.s3a.max.total.tasks":"32","fs.s3a.readahead.range":"64K","hadoop.http.authentication.simple.anonymous.allowed":"true","yarn.nodemanager.master-hostname-resolution-check.enable":"true","fs.s3a.attempts.maximum":"5","yarn.federation.amrmproxy.allocation.history.max.entry":"100","hadoop.registry.zk.connection.timeout.ms":"15000","yarn.resourcemanager.delegation-token-renewer.thread-count":"*********(redacted)","yarn.resourcemanager.delegation-token-renewer.thread-timeout":"*********(redacted)","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","yarn.nodemanager.aux-services.manifest.reload-ms":"0","yarn.nodemanager.emit-container-events":"true","yarn.resourcemanager.resource-profiles.enabled":"false","yarn.timeline-service.hbase-schema.prefix":"prod.","fs.azure.authorization":"false","mapreduce.map.log.level":"INFO","ha.failover-controller.active-standby-elector.zk.op.retries":"3","yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs":"20","hadoop.job.history.user.location":"none","mapreduce.output.fileoutputformat.compress.type":"BLOCK","yarn.resourcemanager.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.timeline-service.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.task.spill.files.count.limit":"-1","mapreduce.ifile.readahead.bytes":"4194304","yarn.sharedcache.app-checker.class":"org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"true","yarn.nodemanager.resource.detect-hardware-capabilities":"false","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","fs.viewfs.overload.scheme.target.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","yarn.federation.gpg.webapp.address":"0.0.0.0:8069","hadoop.security.group.mapping.ldap.search.group.hierarchy.levels":"0","yarn.resourcemanager.fs.state-store.retry-interval-ms":"1000","hadoop.proxyuser.hadoop.hosts":"*","file.stream-buffer-size":"4096","yarn.resourcemanager.application-timeouts.monitor.interval-ms":"3000","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.SnappyCodec","mapreduce.map.speculative":"true","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.hdfs-hash-file":"/runc-root/image-tag-to-hash","mapreduce.job.speculative.retry-after-speculate":"15000","yarn.federation.failover.random.order":"false","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.app.mapreduce.am.container.log.backups":"0","yarn.app.mapreduce.am.log.level":"INFO","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.ImageTagToManifestPlugin","io.bytes.per.checksum":"512","mapreduce.job.reduce.slowstart.completedmaps":"0.05","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices":"auto","yarn.timeline-service.client.internal-timers-ttl-secs":"420","io.compression.codec.zstd.buffersize":"0","fs.s3a.select.output.csv.quote.character":"\"","hadoop.http.logs.enabled":"true","fs.s3a.block.size":"32M","yarn.sharedcache.client-server.address":"0.0.0.0:8045","yarn.nodemanager.logaggregation.threadpool-size-max":"100","yarn.resourcemanager.hostname":"0.0.0.0","ipc.callqueue.overflow.trigger.failover":"false","yarn.resourcemanager.delegation.key.update-interval":"86400000","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"3072","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","yarn.nodemanager.disk-health-checker.enable":"true","fs.s3a.select.output.csv.quote.fields":"always","ipc.client.tcpnodelay":"true","ipc.client.rpc-timeout.ms":"120000","yarn.nodemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","yarn.resourcemanager.delegation-token-renewer.thread-retry-max-attempts":"*********(redacted)","ipc.client.low-latency":"false","yarn.scheduler.skip.node.multiplier":"2","mapreduce.input.lineinputformat.linespermap":"1","yarn.router.interceptor.user.threadpool-size":"5","ipc.client.connect.max.retries.on.timeouts":"5","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"900000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.timeline-service.entity-group-fs-store.app-cache-size":"10","yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-timeout-interval-secs":"360","fs.s3a.socket.recv.buffer":"8192","ipc.backoff.enable":"false","rpc.metrics.timeunit":"MILLISECONDS","mapreduce.output.fs.optimized.committer.enabled":"true","yarn.resourcemanager.resource-tracker.address":"${yarn.resourcemanager.hostname}:8031","yarn.nodemanager.node-labels.provider.fetch-timeout-ms":"1200000","mapreduce.job.heap.memory-mb.ratio":"0.8","yarn.resourcemanager.leveldb-state-store.compaction-interval-secs":"3600","yarn.resourcemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","yarn.nodemanager.pluggable-device-framework.enabled":"false","yarn.scheduler.configuration.fs.path":"file://${hadoop.tmp.dir}/yarn/system/schedconf","dfs.datanode.max.transfer.threads":"4096","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"org.apache.hadoop.security.HttpCrossOriginFilterInitializer","mapreduce.reduce.memory.mb":"6144","mapreduce.admin.user.env":"LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:/usr/lib/hadoop-lzo/lib/native","yarn.timeline-service.hostname":"0.0.0.0","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","yarn.nodemanager.container-metrics.period-ms":"-1","mapreduce.fileoutputcommitter.task.cleanup.enabled":"false","hadoop.proxyuser.oozie.hosts":"*","yarn.nodemanager.log.retain-seconds":"10800","yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds":"3600","ipc.[port_number].callqueue.impl":"java.util.concurrent.LinkedBlockingQueue","yarn.resourcemanager.keytab":"/etc/krb5.keytab","hadoop.security.group.mapping.providers.combined":"true","mapreduce.reduce.merge.inmem.threshold":"1000","yarn.timeline-service.recovery.enabled":"false","fs.azure.saskey.usecontainersaskeyforallaccess":"true","yarn.sharedcache.nm.uploader.thread-count":"20","mapreduce.fileoutputcommitter.cleanup-failures.ignored.emr_internal_use_only.EmrFileSystem":"true","yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs":"3600","ipc.[port_number].weighted-cost.lockfree":"1","mapreduce.shuffle.ssl.enabled":"false","yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds":"259200000","mapreduce.jvm.add-opens-as-default":"true","yarn.nodemanager.opportunistic-containers-max-queue-length":"0","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","mapreduce.job.speculative.minimum-allowed-tasks":"10","fs.s3a.aws.credentials.provider":"software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider","yarn.log-aggregation.retain-seconds":"-1","yarn.router.interceptor.allow-partial-result.enable":"false","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.http.cross-origin.allowed-methods":"GET,POST,HEAD","mapreduce.task.ping-for-liveliness-check.enabled":"false","yarn.resourcemanager.opportunistic-container-allocation.enabled":"false","fs.azure.enable.readahead":"true","mapreduce.jobhistory.webapp.address":"0.0.0.0:19888","hadoop.system.tags":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n      ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.federation.gpg.webapp.cross-origin.enabled":"false","yarn.log-aggregation.file-controller.TFile.class":"org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"${yarn.resourcemanager.hostname}:8088","mapreduce.jobhistory.recovery.enable":"false","mapreduce.reduce.shuffle.parallelcopies":"20","fs.AbstractFileSystem.webhdfs.impl":"org.apache.hadoop.fs.WebHdfs","fs.trash.interval":"0","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","dfs.datanode.du.reserved":"536870912","yarn.app.mapreduce.am.resource.mb":"6144","mapreduce.input.fileinputformat.list-status.num-threads":"1","io.compression.codec.lzo.class":"com.hadoop.compression.lzo.LzoCodec","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","yarn.router.interceptor.user-thread-pool.minimum-pool-size":"5","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.resourcemanager.nm-container-queuing.min-queue-length":"5","yarn.nodemanager.resource.cpu-vcores":"-1","yarn.federation.state-store.clean-up-retry-sleep-time":"1s","mapreduce.job.reduces":"7","fs.s3a.multipart.size":"64M","mapreduce.fileoutputcommitter.algorithm.version.emr_internal_use_only.EmrFileSystem":"2","fs.s3a.select.input.csv.comment.marker":"#","yarn.scheduler.minimum-allocation-vcores":"1","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","mapreduce.job.encrypted-intermediate-data-key-size-bits":"128","fs.iostatistics.thread.level.enabled":"true","ipc.[port_number].weighted-cost.response":"1","yarn.nodemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","ha.health-monitor.sleep-after-disconnect.ms":"1000","yarn.app.mapreduce.shuffle.log.limit.kb":"0","hadoop.security.group.mapping":"org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback","yarn.client.application-client-protocol.poll-timeout-ms":"-1","mapreduce.jobhistory.jhist.format":"binary","mapreduce.task.stuck.timeout-ms":"600000","yarn.resourcemanager.application.max-tag.length":"100","yarn.resourcemanager.ha.enabled":"false","dfs.client.ignore.namenode.default.kms.uri":"false","hadoop.http.staticuser.user":"dr.who","mapreduce.task.exit.timeout.check-interval-ms":"20000","mapreduce.jobhistory.intermediate-user-done-dir.permissions":"770","mapreduce.task.exit.timeout":"60000","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","yarn.resourcemanager.reservation-system.enable":"false","mapreduce.map.output.compress":"true","ha.zookeeper.acl":"world:anyone:rwcda","yarn.federation.state-store.sql.conn-time-out":"10s","hadoop.proxyuser.presto.groups":"*","io.compression.codec.lz4.buffersize":"262144","ipc.server.max.connections":"0","yarn.nodemanager.runtime.linux.docker.default-container-network":"host","yarn.router.webapp.address":"0.0.0.0:8089","yarn.scheduler.maximum-allocation-mb":"8192","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.sharedcache.cleaner.period-mins":"1440","ipc.client.async.calls.max":"100","yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint":"http://localhost:3476/v1.0/docker/cli","yarn.app.mapreduce.am.container.log.limit.kb":"0","ipc.client.connect.retry.interval":"1000","yarn.timeline-service.http-cross-origin.enabled":"false","fs.wasbs.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure","yarn.resourcemanager.nodemanagers.heartbeat-interval-max-ms":"1000","hadoop.http.jmx.nan-filter.enabled":"false","yarn.router.scheduled.executor.threads":"1","hadoop.proxyuser.httpfs.groups":"*","yarn.federation.subcluster-resolver.class":"org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","fs.s3a.select.input.csv.field.delimiter":",","yarn.nodemanager.least-load-policy-selector.multiplier":"50000","mapreduce.jobhistory.cleaner.enable":"true","yarn.timeline-service.client.fd-flush-interval-secs":"10","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","hadoop.proxyuser.httpfs.hosts":"*","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms":"1000","fs.s3a.committer.staging.tmp.path":"tmp/staging","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms":"10","mapreduce.job.end-notification.retry.attempts":"0","yarn.nodemanager.resource.count-logical-processors-as-cores":"false","mapred.output.direct.NativeS3FileSystem":"true","hadoop.registry.zk.root":"/registry","yarn.federation.state-store.sql.pool-name":"YARN-Federation-DataBasePool","adl.feature.ownerandgroup.enableupn":"false","yarn.resourcemanager.zk-max-znode-size.bytes":"1048576","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","yarn.nodemanager.localizer.cache.target-size-mb":"10240","fs.s3a.committer.staging.conflict-mode":"append","mapreduce.client.libjars.wildcard":"true","fs.s3a.committer.staging.unique-filenames":"true","yarn.nodemanager.node-attributes.provider.fetch-timeout-ms":"1200000","yarn.nodemanager.amrmproxy.wait.uam-register.done":"false","fs.s3a.list.version":"2","ftp.client-write-packet-size":"65536","yarn.federation.gpg.policy.generator.load-based.pending.minimum":"100","ipc.[port_number].weighted-cost.lockexclusive":"100","fs.AbstractFileSystem.adl.impl":"org.apache.hadoop.fs.adl.Adl","hadoop.proxyuser.hive.hosts":"*","yarn.nodemanager.container-log-monitor.enable":"false","hadoop.security.key.default.cipher":"AES/CTR/NoPadding","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"24h","fs.s3.maxConnections":"500","mapreduce.job.local-fs.single-disk-limit.check.interval-ms":"5000","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","yarn.nodemanager.amrmproxy.address":"0.0.0.0:8049","ipc.server.listen.queue.size":"256","ipc.[port_number].decay-scheduler.period-ms":"5000","yarn.nodemanager.container-localizer.java.opts.add-exports-as-default":"true","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.cache-refresh-interval-secs":"60","map.sort.class":"org.apache.hadoop.util.QuickSort","yarn.federation.state-store.max-applications":"1000","fs.viewfs.rename.strategy":"SAME_MOUNTPOINT","hadoop.security.kms.client.authentication.retry-count":"1","fs.permissions.umask-mode":"022","fs.s3a.assumed.role.credentials.provider":"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider","yarn.nodemanager.runtime.linux.runc.privileged-containers.allowed":"false","ipc.server.handler.queue.size":"100","yarn.nodemanager.vmem-check-enabled":"true","yarn.nodemanager.numa-awareness.enabled":"false","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.interval-seconds":"60","yarn.federation.registry.base-dir":"yarnfederation/","yarn.nodemanager.health-checker.run-before-startup":"false","spark.app.id":"spark-application-1761565571107","mapreduce.job.max.map":"-1","mapreduce.job.local-fs.single-disk-limit.bytes":"-1","mapreduce.shuffle.pathcache.concurrency-level":"16","mapreduce.job.ubertask.maxreduces":"1","mapreduce.shuffle.pathcache.max-weight":"10485760","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","mapreduce.jobhistory.jobname.limit":"50","fs.s3a.select.input.compression":"none","mapreduce.application.classpath":"\n    $HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*,\n    $HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*,\n    /usr/lib/hadoop-lzo/lib/*,\n    /usr/share/aws/emr/emrfs/conf,\n    /usr/share/aws/emr/emrfs/lib/*,\n    /usr/share/aws/emr/emrfs/auxlib/*,\n    /usr/share/aws/emr/lib/*,\n    /usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar,\n    /usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar,\n    /usr/share/aws/emr/cloudwatch-sink/lib/*,\n    /usr/share/aws/aws-java-sdk/*\n  ","yarn.client.nodemanager-connect.retry-interval-ms":"10000","ipc.[port_number].scheduler.priority.levels":"4","yarn.timeline-service.state-store-class":"org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ","yarn.federation.state-store.clean-up-retry-count":"1","yarn.sharedcache.nested-level":"3","yarn.timeline-service.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","fs.azure.user.agent.prefix":"unknown","yarn.resourcemanager.zk-delegation-token-node.split-index":"*********(redacted)","yarn.nodemanager.numa-awareness.read-topology":"false","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","yarn.registry.class":"org.apache.hadoop.registry.client.impl.FSRegistryOperationsService","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","yarn.scheduler.queue-placement-rules":"user-group","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","yarn.resourcemanager.recovery.enabled":"false","fs.s3a.select.input.csv.header":"none","yarn.federation.gpg.policy.generator.load-based.scaling":"LINEAR","yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-size":"500","yarn.timeline-service.webapp.rest-csrf.enabled":"false","dfs.datanode.available-space-volume-choosing-policy.balanced-space-threshold":"10737418240","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-watermark-high-mb":"0"},"System Properties":{"java.io.tmpdir":"/tmp","RDS_TRUSTSTORE_URL":"file:/opt/amazon/certs/InternalAndExternalAndAWSTrustStore.jks","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","sun.cpu.endian":"little","java.specification.maintenance.version":"1","java.specification.version":"17","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Amazon.com Inc.","java.vm.specification.version":"17","user.home":"/home/hadoop","sun.arch.data.model":"64","sun.boot.library.path":"/usr/lib/jvm/java-17-amazon-corretto.x86_64/lib","user.dir":"/tmp","java.library.path":":/usr/lib/hadoop/lib/native:/usr/lib/hadoop-lzo/lib/native:/docker/usr/lib/hadoop/lib/native:/docker/usr/lib/hadoop-lzo/lib/native:/usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib","os.arch":"amd64","java.vm.version":"17.0.16+8-LTS","java.runtime.version":"17.0.16+8-LTS","java.vm.info":"mixed mode, sharing","java.runtime.name":"OpenJDK Runtime Environment","java.version.date":"2025-07-15","file.separator":"/","java.class.version":"61.0","java.specification.name":"Java Platform API Specification","file.encoding":"UTF-8","user.timezone":"GMT","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","java.vm.compressedOopsMode":"Zero based","os.version":"5.10.244-240.965.amzn2.x86_64","native.encoding":"UTF-8","java.vm.specification.vendor":"Oracle Corporation","javax.net.ssl.trustStore":"/opt/amazon/certs/InternalAndExternalAndAWSTrustStore.jks","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.version":"Corretto-17.0.16.8.1","RDS_ROOT_CERT_PATH":"/opt/amazon/certs/rds-combined-ca-bundle.pem","REDSHIFT_ROOT_CERT_PATH":"/opt/amazon/certs/redshift-ssl-ca-cert.pem","java.vendor.url":"https://aws.amazon.com/corretto/","javax.net.ssl.trustStoreType":"JKS","log4j.configurationFile":"/tmp/glue-job-14058754211823993811/glue-1346396249178199233log4j2.properties","os.name":"Linux","java.vm.vendor":"Amazon.com Inc.","sun.net.httpserver.maxRspTime":"600","jdk.debug":"release","java.vendor.url.bug":"https://github.com/corretto/corretto-17/issues/","user.name":"hadoop","sun.net.httpserver.maxReqTime":"60","java.vm.name":"OpenJDK 64-Bit Server VM","sun.java.command":"*********(redacted)","java.home":"/usr/lib/jvm/java-17-amazon-corretto.x86_64","javax.net.ssl.trustStorePassword":"*********(redacted)","java.version":"17.0.16","sun.io.unicode.encoding":"UnicodeLittle"},"Metrics Properties":{"driver.source.AWSDILyra.class":"org.apache.spark.metrics.source.AWSDILyraSource","*.source.system.class":"org.apache.spark.metrics.source.SystemMetricsSource","driver.source.glue.jobPerformance.class":"org.apache.spark.metrics.source.PerformanceMetricsSource","*.source.s3.class":"org.apache.spark.metrics.source.S3FileSystemSource","*.sink.servlet.path":"/metrics/json","*.sink.GlueCloudwatch.proxyPort":"-1","driver.source.throughput.class":"org.apache.spark.metrics.source.ThroughputMetricsSource","driver.source.aggregate.class":"org.apache.spark.metrics.source.AggregateMetricsSource","*.sink.GlueCloudwatch.glueVersion":"5.0","*.sink.GlueCloudwatch.legacyMetrics":"true","*.sink.servlet.class":"org.apache.spark.metrics.sink.MetricsServlet","driver.sink.AWSDILyra.class":"org.apache.spark.metrics.sink.AWSDILyraSink","driver.source.glue.jobPerformance.skewnessFactor":"5","driver.source.glue.resourceUtilization.class":"org.apache.spark.metrics.source.ResourceUtilizationMetricsSource","*.sink.GlueCloudwatch.namespace":"Glue","*.sink.GlueCloudwatch.class":"org.apache.spark.metrics.sink.GlueCloudwatchSink","*.sink.GlueCloudwatch.accountId":"896271372615","*.sink.GlueCloudwatch.proxyHost":"null","*.source.jvm.class":"org.apache.spark.metrics.source.JvmSource","applications.sink.servlet.path":"/metrics/applications/json","master.sink.servlet.path":"/metrics/master/json","*.sink.GlueCloudwatch.jobName":"test_zodiac_pg_to_redshift","*.sink.GlueCloudwatch.jobRunId":"jr_4464f784f631b1a0c7231da55ee1a6f368615377dff170f59beab638b8587351","*.source.glue.verticalScaling.class":"org.apache.spark.metrics.source.VerticalScalingMetricsSource"},"Classpath Entries":{"/usr/lib/hadoop/hadoop-annotations.jar":"System Classpath","/usr/lib/hadoop/jars/*":"System Classpath","/usr/lib/spark/jars/aircompressor-0.27.jar":"System Classpath","/usr/lib/spark/jars/arrow-format-12.0.1.jar":"System Classpath","/usr/lib/spark/jars/RoaringBitmap-0.9.45.jar":"System Classpath","/usr/lib/hadoop/lib/metrics-core-3.2.4.jar":"System Classpath","/usr/lib/hadoop/lib/avro-1.9.2.jar":"System Classpath","/usr/lib/spark/jars/netty-all-4.1.100.Final.jar":"System Classpath","/usr/lib/spark/jars/jackson-core-asl-1.9.13.jar":"System Classpath","/usr/lib/spark/jars/spark-acl-1.3.0.jar":"System Classpath","/usr/share/aws/aws-glue-shuffle/AWSGlueShuffle-5.4.0.jar":"System Classpath","/usr/lib/hadoop/lib/commons-net-3.9.0.jar":"System Classpath","/usr/lib/hadoop/hadoop-streaming-3.4.1-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/jackson-datatype-jsr310-2.15.2.jar":"System Classpath","/usr/lib/spark/jars/netty-buffer-4.1.100.Final.jar":"System Classpath","/usr/lib/hadoop/lib/commons-compress-1.26.1.jar":"System Classpath","/usr/lib/hadoop/lib/commons-beanutils-1.9.4.jar":"System Classpath","spark://172.36.69.93:46463/jars/5qwJmK-aws-glue-di-package-5.0.731.jar":"Added By User","/usr/lib/spark/jars/aopalliance-repackaged-2.6.1.jar":"System Classpath","/usr/lib/hadoop/hadoop-registry-3.4.1-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/metrics-jmx-4.2.19.jar":"System Classpath","/usr/share/aws/iceberg/lib/iceberg-emr-common.jar":"System Classpath","/usr/share/aws/delta/lib/delta-flink-shadeddeps-3.3.0-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/netty-transport-native-kqueue-4.1.100.Final-osx-x86_64.jar":"System Classpath","/usr/lib/spark/jars/zookeeper-jute-3.9.1.jar":"System Classpath","/usr/lib/hadoop/hadoop-gridmix.jar":"System Classpath","/usr/share/java/Hive-JSON-Serde/json-udf.jar":"System Classpath","/usr/lib/spark/jars/osgi-resource-locator-1.0.3.jar":"System Classpath","/usr/share/aws/redshift/spark-redshift/lib/spark-avro_2.12-3.5.4-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/hive-storage-api-2.8.1.jar":"System Classpath","/usr/lib/hadoop/lib/nimbus-jose-jwt-9.37.2.jar":"System Classpath","/usr/lib/spark/jars/commons-math3-3.6.1.jar":"System Classpath","/usr/lib/spark/jars/libfb303-0.9.3.jar":"System Classpath","/usr/lib/hadoop/hadoop-extras-3.4.1-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/hive-llap-common-2.3.9-amzn-4.jar":"System Classpath","/usr/share/aws/emr/emrfs/lib/emrfs-hadoop-assembly-2.69.0.jar":"System Classpath","/usr/lib/spark/jars/netty-transport-classes-epoll-4.1.100.Final.jar":"System Classpath","/usr/lib/spark/jars/flatbuffers-java-1.12.0.jar":"System Classpath","/usr/lib/spark/jars/datanucleus-rdbms-4.1.19.jar":"System Classpath","/usr/lib/spark/jars/jersey-hk2-2.40.jar":"System Classpath","/usr/lib/hadoop/lib/netty-codec-http-4.1.100.Final.jar":"System Classpath","/usr/lib/hadoop/hadoop-azure.jar":"System Classpath","/usr/lib/spark/jars/spark-unsafe_2.12-3.5.4-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/jakarta.inject-2.6.1.jar":"System Classpath","/usr/lib/hadoop/lib/commons-io-2.16.1.jar":"System Classpath","/usr/share/aws/delta/lib/delta-storage-s3-dynamodb.jar":"System Classpath","/docker/usr/share/aws/emr/goodies/lib/emr-spark-goodies.jar":"System Classpath","/usr/share/aws/iceberg/lib/iceberg-hive-runtime-1.7.1-amzn-1.jar":"System Classpath","/usr/lib/spark/jars/animal-sniffer-annotations-1.23.jar":"System Classpath","/usr/lib/hadoop/lib/jsp-api-2.1.jar":"System Classpath","/usr/lib/hadoop/hadoop-rumen-3.4.1-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/jsr305-3.0.0.jar":"System Classpath","/tmp/glue-job-14058754211823993811/*":"System Classpath","/usr/lib/spark/jars/HikariCP-2.5.1.jar":"System Classpath","/usr/lib/spark/jars/jdo-api-3.0.1.jar":"System Classpath","/usr/lib/spark/jars/pickle-1.3.jar":"System Classpath","/usr/lib/hadoop/lib/jettison-1.5.4.jar":"System Classpath","/usr/lib/hadoop/lib/jersey-servlet-1.19.4.jar":"System Classpath","/usr/lib/hadoop/lib/re2j-1.1.jar":"System Classpath","/usr/lib/spark/jars/JTransforms-3.1.jar":"System Classpath","/usr/lib/spark/jars/xbean-asm9-shaded-4.23.jar":"System Classpath","/usr/lib/hadoop/lib/netty-codec-mqtt-4.1.100.Final.jar":"System Classpath","/usr/share/aws/delta/lib/delta-spark_2.12-3.3.0-amzn-0.jar":"System Classpath","/usr/lib/hadoop/hadoop-resourceestimator.jar":"System Classpath","/usr/share/java/Hive-JSON-Serde/hive-openx-serde.jar":"System Classpath","/usr/lib/spark/jars/jersey-common-2.40.jar":"System Classpath","/usr/lib/hadoop/lib/netty-transport-native-kqueue-4.1.100.Final-osx-x86_64.jar":"System Classpath","/usr/lib/spark/jars/libthrift-0.12.0.jar":"System Classpath","/usr/lib/spark/jars/httpcore-4.4.13.jar":"System Classpath","/usr/lib/hadoop/lib/netty-transport-4.1.100.Final.jar":"System Classpath","/usr/lib/hadoop/lib/netty-codec-4.1.100.Final.jar":"System Classpath","/usr/lib/spark/jars/netty-codec-socks-4.1.100.Final.jar":"System Classpath","/usr/lib/hadoop/lib/netty-transport-sctp-4.1.100.Final.jar":"System Classpath","/usr/share/aws/emr/emrfs/lib/jsr305-3.0.2.jar":"System Classpath","/usr/lib/spark/jars/jpam-1.1.jar":"System Classpath","/usr/share/aws/hmclient/lib/aws-glue-datacatalog-spark-client.jar":"System Classpath","/usr/lib/spark/jars/kryo-shaded-4.0.2.jar":"System Classpath","/usr/lib/spark/jars/scala-collection-compat_2.12-2.7.0.jar":"System Classpath","/usr/lib/spark/jars/scala-compiler-2.12.18.jar":"System Classpath","/usr/lib/spark/jars/netty-common-4.1.100.Final.jar":"System Classpath","/usr/lib/spark/jars/py4j-0.10.9.7.jar":"System Classpath","/usr/lib/spark/jars/arrow-memory-netty-12.0.1.jar":"System Classpath","/usr/lib/hadoop/lib/bcprov-jdk18on-1.78.1.jar":"System Classpath","/usr/lib/spark/jars/curator-client-2.13.0.jar":"System Classpath","/usr/lib/spark/jars/orc-mapreduce-1.9.5-shaded-protobuf.jar":"System Classpath","/usr/lib/hadoop/lib/jetty-util-9.4.53.v20231009.jar":"System Classpath","/usr/lib/hadoop/hadoop-shaded-guava-1.3.0.jar":"System Classpath","/usr/share/aws/sagemaker-spark-sdk/lib/sagemaker-spark-sdk.jar":"System Classpath","/usr/lib/spark/jars/chill-java-0.10.0.jar":"System Classpath","/usr/lib/spark/jars/curator-framework-2.13.0.jar":"System Classpath","/usr/share/aws/emr/security/conf":"System Classpath","/usr/share/aws/emr/emrfs/lib/kryo-shaded-4.0.2.jar":"System Classpath","/usr/lib/spark/jars/istack-commons-runtime-3.0.8.jar":"System Classpath","/usr/lib/hadoop/lib/zookeeper-d*.jar":"System Classpath","/usr/lib/spark/jars/hk2-locator-2.6.1.jar":"System Classpath","/usr/lib/spark/jars/commons-lang-2.6.jar":"System Classpath","/usr/lib/hadoop/hadoop-common-3.4.1-amzn-0.jar":"System Classpath","/usr/lib/hadoop/lib/netty-codec-xml-4.1.100.Final.jar":"System Classpath","/usr/lib/hadoop/lib/javax.servlet-api-3.1.0.jar":"System Classpath","/usr/lib/spark/jars/commons-collections4-4.4.jar":"System Classpath","/usr/lib/livy/repl_2.12-jars/*":"System Classpath","/usr/lib/hadoop/lib/jsr305-3.0.2.jar":"System Classpath","/usr/share/aws/emr/goodies/lib/emr-serverless-spark-goodies.jar":"System Classpath","/usr/lib/spark/jars/curator-recipes-2.13.0.jar":"System Classpath","/usr/lib/hadoop/hadoop-gridmix-3.4.1-amzn-0.jar":"System Classpath","/usr/share/aws/emr/emrfs/lib/javax.inject-1.jar":"System Classpath","/usr/lib/spark/jars/jersey-server-2.40.jar":"System Classpath","/usr/lib/spark/jars/jackson-dataformat-yaml-2.15.2.jar":"System Classpath","/usr/lib/hadoop/hadoop-dynamometer-blockgen-3.4.1-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/jackson-annotations-2.15.2.jar":"System Classpath","/tmp/glue-job-14058754211823993811/jars/5qwJmK-aws-glue-di-package-5.0.731.jar":"System Classpath","/usr/share/aws/iceberg/lib/iceberg-spark3-runtime.jar":"System Classpath","/usr/lib/spark/jars/metrics-jvm-4.2.19.jar":"System Classpath","/usr/lib/spark/jars/ivy-2.5.1.jar":"System Classpath","/usr/lib/spark/jars/httpclient-4.5.13.jar":"System Classpath","/usr/lib/spark/jars/jersey-container-servlet-core-2.40.jar":"System Classpath","/usr/lib/hadoop/lib/jakarta.activation-api-1.2.1.jar":"System Classpath","/usr/lib/hadoop/lib/kerby-asn1-2.0.3.jar":"System Classpath","/usr/lib/hadoop/lib/jetty-http-9.4.53.v20231009.jar":"System Classpath","/usr/lib/spark/jars/jackson-databind-2.15.2.jar":"System Classpath","/usr/lib/spark/jars/paranamer-2.8.jar":"System Classpath","/usr/share/java/Hive-JSON-Serde/json-udf-1.3.8.jar":"System Classpath","/usr/share/aws/emr/emrfs/lib/AssumeRoleAWSCredentialsProvider-1.1-SNAPSHOT.jar":"System Classpath","/usr/share/aws/hmclient/lib/aws-glue-datacatalog-hive3-client-4.5.0.jar":"System Classpath","/usr/lib/spark/jars/rocksdbjni-8.3.2.jar":"System Classpath","/usr/lib/hadoop/hadoop-fs2img-3.4.1-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/snappy-java-1.1.10.5.jar":"System Classpath","/usr/lib/spark/jars/spark-hive-thriftserver_2.12-3.5.4-amzn-0.jar":"System Classpath","/usr/lib/hadoop/lib/netty-codec-socks-4.1.100.Final.jar":"System Classpath","/docker/usr/share/aws/sagemaker-spark-sdk/lib/sagemaker-spark-sdk.jar":"System Classpath","/usr/lib/spark/jars/jakarta.annotation-api-1.3.5.jar":"System Classpath","/usr/lib/spark/jars/jersey-client-2.40.jar":"System Classpath","/usr/lib/spark/jars/commons-collections-3.2.2.jar":"System Classpath","/usr/share/aws/delta/lib/delta-flink.jar":"System Classpath","/usr/share/aws/emr/emrfs/lib/findbugs-annotations-3.0.1.jar":"System Classpath","/usr/share/aws/glue-connectors/lib/GlueSparkConnector-OpenSearch-1.0.jar":"System Classpath","/usr/lib/spark/jars/commons-logging-1.1.3.jar":"System Classpath","/usr/lib/hadoop/lib/jsr311-api-1.1.1.jar":"System Classpath","/usr/lib/spark/jars/jackson-module-scala_2.12-2.15.2.jar":"System Classpath","/usr/lib/spark/jars/commons-compiler-3.1.9.jar":"System Classpath","/usr/share/aws/glue-pii-dependencies/lib/stanford-corenlp-3.6.0-models.jar":"System Classpath","/usr/lib/spark/jars/hive-metastore-2.3.9-amzn-4.jar":"System Classpath","/usr/lib/spark/jars/hadoop-client-runtime-3.4.1-amzn-0.jar":"System Classpath","/tmp/glue-job-14058754211823993811/extra-jars/*":"System Classpath","/usr/lib/hadoop/lib/netty-codec-redis-4.1.100.Final.jar":"System Classpath","/usr/lib/hadoop/lib/jetty-security-9.4.53.v20231009.jar":"System Classpath","/usr/lib/hadoop/lib/httpcore-4.4.13.jar":"System Classpath","/usr/lib/spark/jars/spark-network-common_2.12-3.5.4-amzn-0.jar":"System Classpath","/usr/lib/hudi/hudi-spark-bundle.jar":"System Classpath","/usr/lib/spark/jars/chill_2.12-0.10.0.jar":"System Classpath","/usr/lib/hadoop/lib/netty-transport-native-unix-common-4.1.100.Final.jar":"System Classpath","/usr/lib/spark/jars/volcano-client-6.7.2.jar":"System Classpath","spark://172.36.69.93:46463/files/glue-job-14058754211823993811_glue_venv.zip#python_environment":"Added By User","/usr/lib/hadoop/hadoop-client.jar":"System Classpath","/usr/lib/hadoop/hadoop-aliyun-3.4.1-amzn-0.jar":"System Classpath","/usr/lib/hadoop/lib/httpclient-4.5.13.jar":"System Classpath","/usr/lib/spark/jars/datasketches-java-3.3.0.jar":"System Classpath","/usr/lib/spark/jars/netty-transport-native-epoll-4.1.100.Final-linux-x86_64.jar":"System Classpath","/usr/lib/spark/jars/jcl-over-slf4j-2.0.7.jar":"System Classpath","/usr/lib/hadoop/hadoop-dynamometer-workload.jar":"System Classpath","/usr/lib/spark/jars/emr-serverless-spark-goodies.jar":"System Classpath","/usr/lib/spark/jars/parquet-hadoop-1.13.1-amzn-6.jar":"System Classpath","/usr/lib/hadoop/lib/netty-transport-native-epoll-4.1.100.Final-linux-x86_64.jar":"System Classpath","/usr/lib/hadoop/hadoop-aws-3.4.1-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/netty-transport-native-epoll-4.1.100.Final-linux-aarch_64.jar":"System Classpath","/usr/share/aws/redshift/jdbc/aws-secretsmanager-jdbc-1.0.12.jar":"System Classpath","/usr/lib/spark/jars/hive-shims-common-2.3.9-amzn-4.jar":"System Classpath","/usr/lib/hadoop-lzo/lib/hadoop-lzo-0.4.19.jar":"System Classpath","/usr/lib/spark/jars/glue-spark-goodies-2.18.0.jar":"System Classpath","/usr/share/aws/redshift/spark-redshift/lib/spark-redshift.jar":"System Classpath","/usr/lib/hadoop/lib/jackson-core-2.12.7.jar":"System Classpath","/usr/lib/hadoop/lib/netty-resolver-dns-classes-macos-4.1.100.Final.jar":"System Classpath","/usr/lib/spark/jars/arpack-3.0.3.jar":"System Classpath","/usr/lib/spark/jars/xz-1.9.jar":"System Classpath","/usr/lib/spark/jars/commons-lang3-3.12.0.jar":"System Classpath","/usr/lib/spark/jars/janino-3.1.9.jar":"System Classpath","/usr/lib/spark/jars/javax.jdo-3.2.0-m3.jar":"System Classpath","/usr/lib/hadoop/hadoop-auth-3.4.1-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/grpc-protobuf-lite-1.56.0.jar":"System Classpath","/usr/share/aws/delta/lib/delta-flink-3.3.0-amzn-0.jar":"System Classpath","/tmp/glue-job-14058754211823993811/aws_glue_connectors/marketplace/*":"System Classpath","/usr/lib/spark/jars/spark-common-utils_2.12-3.5.4-amzn-0.jar":"System Classpath","/usr/share/aws/emr/emrfs/lib/minlog-1.3.0.jar":"System Classpath","/usr/share/aws/emr/security/lib/*":"System Classpath","/usr/lib/spark/jars/avro-mapred-1.11.2-cve-2024-47561-amzn-1.jar":"System Classpath","/usr/lib/spark/jars/snakeyaml-engine-2.6.jar":"System Classpath","/usr/lib/spark/jars/jta-1.1.jar":"System Classpath","/usr/lib/hadoop/lib/zookeeper-3.9.3-amzn-0.jar":"System Classpath","/usr/lib/hadoop/hadoop-azure-datalake.jar":"System Classpath","/usr/lib/spark/jars/spark-sql-api_2.12-3.5.4-amzn-0.jar":"System Classpath","/usr/lib/hadoop/lib/jul-to-slf4j-1.7.36.jar":"System Classpath","/usr/lib/spark/jars/jackson-core-2.15.2.jar":"System Classpath","/usr/lib/hudi/hudi-spark3-bundle_2.12-0.15.0-amzn-5.jar":"System Classpath","/usr/lib/spark/jars/json-1.8.jar":"System Classpath","/usr/lib/spark/jars/disruptor-3.3.7.jar":"System Classpath","/tmp/glue-job-14058754211823993811/python/*":"System Classpath","/usr/lib/spark/jars/annotations-17.0.0.jar":"System Classpath","/usr/share/aws/redshift/jdbc/AwsSecretsManagerJDBC.jar":"System Classpath","/usr/lib/spark/jars/stream-2.9.6.jar":"System Classpath","/usr/lib/hadoop/hadoop-dynamometer-blockgen.jar":"System Classpath","/usr/share/aws/redshift/jdbc/redshift-jdbc42-2.1.0.29.jar":"System Classpath","/usr/lib/spark/jars/scala-xml_2.12-2.1.0.jar":"System Classpath","/usr/lib/hadoop/lib/netty-resolver-dns-4.1.100.Final.jar":"System Classpath","/usr/lib/hadoop/lib/netty-transport-classes-epoll-4.1.100.Final.jar":"System Classpath","/usr/lib/hadoop/hadoop-shaded-protobuf_3_25-1.3.0.jar":"System Classpath","/usr/lib/spark/jars/log4j-slf4j2-impl-2.20.0.jar":"System Classpath","/usr/lib/spark/jars/emr-serverless-goodies-common.jar":"System Classpath","/usr/lib/hadoop/hadoop-archive-logs.jar":"System Classpath","/usr/lib/spark/jars/hive-beeline-2.3.9-amzn-4.jar":"System Classpath","/usr/lib/hadoop/lib/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar":"System Classpath","/usr/lib/hadoop/hadoop-resourceestimator-3.4.1-amzn-0.jar":"System Classpath","/usr/lib/hadoop/lib/jetty-webapp-9.4.53.v20231009.jar":"System Classpath","/usr/lib/spark/jars/aggdesigner-algorithm-6.0.jar":"System Classpath","/usr/lib/hadoop/lib/slf4j-reload4j-1.7.36.jar":"System Classpath","/usr/share/aws/hmclient/lib/aws-glue-datacatalog-spark-client-4.5.0.jar":"System Classpath","/usr/lib/hadoop/lib/netty-codec-http2-4.1.100.Final.jar":"System Classpath","/usr/lib/spark/jars/volcano-model-v1beta1-6.7.2.jar":"System Classpath","/usr/lib/spark/jars/spark-fgac_2.12-3.5.4-amzn-0.jar":"System Classpath","/usr/lib/hadoop/lib/jsch-0.1.55.jar":"System Classpath","/usr/lib/hadoop/lib/woodstox-core-5.4.0.jar":"System Classpath","/usr/share/aws/glue-connectors/lib/GlueSparkConnector-AzureSQL-1.0.jar":"System Classpath","/usr/share/aws/glue-connectors/lib/GlueSparkConnector-PostgreSQL-1.0.jar":"System Classpath","/usr/lib/hadoop/lib/netty-codec-haproxy-4.1.100.Final.jar":"System Classpath","/usr/lib/hadoop/lib/jetty-io-9.4.53.v20231009.jar":"System Classpath","/usr/lib/hadoop/hadoop-sls.jar":"System Classpath","/usr/lib/hadoop/hadoop-datajoin-3.4.1-amzn-0.jar":"System Classpath","/usr/lib/hadoop/lib/jackson-annotations-2.12.7.jar":"System Classpath","/usr/lib/spark/jars/antlr4-runtime-4.9.3.jar":"System Classpath","/usr/lib/spark/jars/parquet-jackson-1.13.1-amzn-6.jar":"System Classpath","/usr/lib/spark/jars/orc-shims-1.9.5.jar":"System Classpath","/usr/lib/spark/jars/remotetea-oncrpc-1.1.2.jar":"System Classpath","/usr/share/aws/iceberg/lib/iceberg-flink-runtime.jar":"System Classpath","/usr/lib/hadoop/lib/netty-common-4.1.100.Final.jar":"System Classpath","/usr/lib/hadoop/lib/netty-codec-stomp-4.1.100.Final.jar":"System Classpath","/usr/lib/hadoop/lib/reload4j-1.2.22.jar":"System Classpath","/usr/lib/spark/jars/okio-1.17.6.jar":"System Classpath","/usr/lib/spark/jars/jline-2.14.6.jar":"System Classpath","/usr/lib/hadoop/lib/animal-sniffer-annotations-1.17.jar":"System Classpath","/usr/lib/hadoop/lib/stax2-api-4.2.1.jar":"System Classpath","/usr/lib/spark/jars/spark-sql_2.12-3.5.4-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/json4s-ast_2.12-3.7.0-M11.jar":"System Classpath","/usr/lib/hadoop/lib/failureaccess-1.0.jar":"System Classpath","/usr/lib/spark/jars/activation-1.1.1.jar":"System Classpath","/usr/lib/hadoop/lib/netty-codec-dns-4.1.100.Final.jar":"System Classpath","/usr/lib/hadoop/lib/checker-qual-2.5.2.jar":"System Classpath","/usr/lib/spark/jars/hive-shims-scheduler-2.3.9-amzn-4.jar":"System Classpath","/usr/lib/spark/jars/spire_2.12-0.17.0.jar":"System Classpath","/usr/lib/hadoop/hadoop-kafka-3.4.1-amzn-0.jar":"System Classpath","/usr/lib/hadoop/hadoop-sls-3.4.1-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/commons-text-1.10.0.jar":"System Classpath","/usr/lib/hadoop/lib/j2objc-annotations-1.1.jar":"System Classpath","/usr/lib/hadoop/lib/kerb-crypto-2.0.3.jar":"System Classpath","/usr/lib/spark/jars/jetty-rewrite-9.3.27.v20190418.jar":"System Classpath","/usr/lib/hadoop/hadoop-kafka.jar":"System Classpath","/usr/lib/spark/jars/hive-serde-2.3.9-amzn-4.jar":"System Classpath","/usr/lib/spark/jars/arrow-vector-12.0.1.jar":"System Classpath","/usr/lib/spark/jars/spark-sketch_2.12-3.5.4-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/spark-graphx_2.12-3.5.4-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/leveldbjni-all-1.8.jar":"System Classpath","/usr/lib/hadoop/hadoop-dynamometer-infra.jar":"System Classpath","/usr/lib/spark/jars/gmetric4j-1.0.10.jar":"System Classpath","/usr/lib/hadoop/lib/kerby-util-2.0.3.jar":"System Classpath","/usr/share/aws/delta/lib/delta-storage-3.3.0-amzn-0.jar":"System Classpath","/usr/lib/hadoop/hadoop-rumen.jar":"System Classpath","/usr/lib/hadoop/lib/kerby-pkix-2.0.3.jar":"System Classpath","/usr/share/aws/*":"System Classpath","/usr/lib/hadoop/hadoop-streaming.jar":"System Classpath","/usr/lib/spark/jars/univocity-parsers-2.9.1.jar":"System Classpath","/usr/lib/spark/jars/netty-codec-4.1.100.Final.jar":"System Classpath","/usr/lib/spark/jars/snakeyaml-2.1.jar":"System Classpath","/usr/lib/spark/jars/javax.servlet-api-3.1.0.jar":"System Classpath","/usr/lib/spark/jars/metrics-core-4.2.19.jar":"System Classpath","/usr/lib/spark/jars/breeze-macros_2.12-2.1.0.jar":"System Classpath","/usr/lib/livy/rsc-jars/*":"System Classpath","/usr/lib/spark/jars/audience-annotations-0.12.0.jar":"System Classpath","/usr/lib/hadoop/hadoop-datajoin.jar":"System Classpath","/docker/usr/share/aws/emr/s3select/lib/emr-s3-select-spark-connector.jar":"System Classpath","/usr/lib/spark/jars/parquet-common-1.13.1-amzn-6.jar":"System Classpath","/usr/share/aws/datazone-openlineage-spark/lib/DataZoneOpenLineageSpark-1.0.jar":"System Classpath","/usr/lib/spark/jars/hive-common-2.3.9-amzn-4.jar":"System Classpath","/usr/share/aws/emr/emrfs/lib/error_prone_annotations-2.1.3.jar":"System Classpath","/usr/lib/hadoop/hadoop-aliyun.jar":"System Classpath","/usr/lib/spark/jars/scala-parser-combinators_2.12-2.3.0.jar":"System Classpath","/usr/lib/hadoop/lib/commons-logging-1.2.jar":"System Classpath","/usr/lib/spark/jars/perfmark-api-0.26.0.jar":"System Classpath","/usr/lib/hadoop/lib/commons-text-1.10.0.jar":"System Classpath","/usr/lib/hadoop/hadoop-dynamometer-infra-3.4.1-amzn-0.jar":"System Classpath","/usr/share/aws/emr-glue-bootstrap/EMRGlueBootstrap-5.3.0-jar-with-dependencies.jar":"System Classpath","/usr/lib/spark/jars/error_prone_annotations-2.18.0.jar":"System Classpath","/usr/lib/spark/jars/transaction-api-1.1.jar":"System Classpath","/usr/lib/spark/jars/spark-mllib-local_2.12-3.5.4-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/derby-10.14.2.0.jar":"System Classpath","/usr/share/aws/glue-connectors/lib/GlueSparkConnector-SQLServer-1.0.jar":"System Classpath","/usr/lib/hadoop/lib/guava-27.0-jre.jar":"System Classpath","/usr/lib/spark/jars/json4s-jackson_2.12-3.7.0-M11.jar":"System Classpath","/usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar":"System Classpath","/usr/lib/spark/jars/spark-repl_2.12-3.5.4-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/spark-launcher_2.12-3.5.4-amzn-0.jar":"System Classpath","spark://172.36.69.93:46463/jars/OBgZGQ-AwsGlueMLLibs.jar":"Added By User","/usr/lib/spark/jars/emr-spark-goodies.jar":"System Classpath","/usr/lib/hudi/hudi-utilities-bundle_2.12-0.15.0-amzn-5.jar":"System Classpath","/docker/usr/share/java/Hive-JSON-Serde/hive-openx-serde.jar":"System Classpath","/usr/lib/spark/jars/grpc-protobuf-1.56.0.jar":"System Classpath","/usr/lib/spark/jars/netty-codec-http-4.1.100.Final.jar":"System Classpath","/usr/share/java/Hive-JSON-Serde/json.jar":"System Classpath","/usr/lib/spark/jars/tink-1.9.0.jar":"System Classpath","/usr/lib/hudi/hudi-utilities-bundle.jar":"System Classpath","/usr/lib/spark/jars/datanucleus-core-4.1.17.jar":"System Classpath","/usr/lib/hadoop/lib/audience-annotations-0.12.0.jar":"System Classpath","/usr/lib/hadoop/lib/curator-framework-5.2.0.jar":"System Classpath","/usr/share/aws/delta/lib/FlinkDeltaReadAndWriteTest.jar":"System Classpath","/usr/share/aws/glue-connectors/lib/GlueSparkConnector-Vertica-1.0.jar":"System Classpath","/usr/lib/spark/jars/netty-handler-4.1.100.Final.jar":"System Classpath","/usr/lib/spark/jars/minlog-1.3.0.jar":"System Classpath","/usr/lib/spark/jars/spark-mllib_2.12-3.5.4-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/jackson-mapper-asl-1.9.13.jar":"System Classpath","/usr/lib/spark/jars/zjsonpatch-0.3.0.jar":"System Classpath","/usr/share/aws/aws-java-sdk-v2/aws-sdk-java-bundle-2.29.52.jar":"System Classpath","/usr/lib/spark/jars/orc-core-1.9.5-shaded-protobuf.jar":"System Classpath","/usr/lib/spark/jars/gson-2.10.1.jar":"System Classpath","/usr/lib/hadoop/lib/jetty-xml-9.4.53.v20231009.jar":"System Classpath","/usr/lib/hadoop/hadoop-fs2img.jar":"System Classpath","/usr/share/aws/glue-connectors/lib/GlueSparkConnector-BigQuery-1.0.jar":"System Classpath","/usr/lib/hadoop/lib/netty-transport-native-epoll-4.1.100.Final-linux-aarch_64.jar":"System Classpath","/usr/lib/spark/jars/jakarta.servlet-api-4.0.3.jar":"System Classpath","/usr/lib/spark/jars/hive-shims-0.23-2.3.9-amzn-4.jar":"System Classpath","/usr/lib/spark/jars/objenesis-3.3.jar":"System Classpath","/usr/lib/spark/jars/netty-resolver-4.1.100.Final.jar":"System Classpath","/usr/lib/hadoop/hadoop-auth.jar":"System Classpath","/docker/usr/share/aws/emr/emrfs/conf":"System Classpath","/usr/lib/spark/jars/hive-exec-2.3.9-amzn-4-core.jar":"System Classpath","/usr/lib/spark/jars/datasketches-memory-2.1.0.jar":"System Classpath","/usr/lib/spark/jars/oro-2.0.8.jar":"System Classpath","/usr/lib/spark/jars/grpc-core-1.56.0.jar":"System Classpath","/usr/lib/spark/jars/scala-reflect-2.12.18.jar":"System Classpath","/usr/lib/hadoop/lib/netty-transport-native-epoll-4.1.100.Final.jar":"System Classpath","/usr/lib/spark/jars/parquet-column-1.13.1-amzn-6.jar":"System Classpath","/usr/lib/hadoop/lib/netty-codec-memcache-4.1.100.Final.jar":"System Classpath","/usr/lib/spark/jars/log4j-1.2-api-2.20.0.jar":"System Classpath","/usr/lib/hadoop/lib/gson-2.9.0.jar":"System Classpath","/docker/usr/share/aws/emr/security/conf":"System Classpath","/usr/share/aws/delta/lib/delta-iceberg.jar":"System Classpath","/usr/lib/hadoop/hadoop-federation-balance-3.4.1-amzn-0.jar":"System Classpath","/usr/lib/hadoop/hadoop-nfs.jar":"System Classpath","/usr/lib/spark/jars/grpc-stub-1.56.0.jar":"System Classpath","/usr/share/aws/delta/lib/flink-example-1.1-SNAPSHOT-jar-with-dependencies.jar":"System Classpath","/usr/lib/hadoop/hadoop-common-3.4.1-amzn-0-tests.jar":"System Classpath","/usr/lib/hadoop/lib/jcip-annotations-1.0-1.jar":"System Classpath","/usr/lib/spark/jars/cats-kernel_2.12-2.1.1.jar":"System Classpath","/usr/lib/spark/jars/joda-time-2.12.5.jar":"System Classpath","/docker/usr/share/aws/emr/emrfs/auxlib/*":"System Classpath","/usr/lib/hadoop/lib/netty-all-4.1.100.Final.jar":"System Classpath","/usr/lib/spark/jars/commons-pool-1.5.4.jar":"System Classpath","/usr/lib/hadoop/hadoop-archive-logs-3.4.1-amzn-0.jar":"System Classpath","/usr/share/aws/delta/lib/delta-spark.jar":"System Classpath","/usr/share/aws/redshift/jdbc/aws-secretsmanager-caching-java-1.0.2.jar":"System Classpath","/usr/lib/spark/jars/bonecp-0.8.0.RELEASE.jar":"System Classpath","/usr/lib/spark/jars/netty-transport-4.1.100.Final.jar":"System Classpath","/usr/lib/spark/jars/hk2-utils-2.6.1.jar":"System Classpath","/usr/share/aws/delta/lib/delta-storage.jar":"System Classpath","/usr/share/aws/redshift/spark-redshift/lib/spark-redshift_2.12-6.4.0-spark_3.5.jar":"System Classpath","/usr/lib/spark/jars/spark-tags_2.12-3.5.4-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/netty-transport-native-kqueue-4.1.100.Final-osx-aarch_64.jar":"System Classpath","/usr/lib/hudi/hudi-aws-bundle-0.15.0-amzn-5.jar":"System Classpath","/usr/share/aws/delta/lib/delta-standalone.jar":"System Classpath","/usr/lib/spark/jars/breeze_2.12-2.1.0.jar":"System Classpath","/usr/lib/spark/jars/commons-io-2.16.1.jar":"System Classpath","/usr/lib/spark/jars/algebra_2.12-2.0.1.jar":"System Classpath","/usr/lib/spark/jars/okhttp-3.12.12.jar":"System Classpath","/usr/share/aws/emr/emrfs/lib/jmespath-java-1.12.705.jar":"System Classpath","/usr/lib/spark/jars/commons-dbcp-1.4.jar":"System Classpath","/usr/share/aws/delta/lib/delta-flink-shadeddeps.jar":"System Classpath","/usr/lib/hadoop/hadoop-registry.jar":"System Classpath","/usr/lib/spark/jars/jersey-container-servlet-2.40.jar":"System Classpath","/usr/lib/spark/jars/netty-codec-http2-4.1.100.Final.jar":"System Classpath","/usr/lib/spark/jars/hadoop-shaded-guava-1.3.0.jar":"System Classpath","/usr/lib/hadoop/lib/netty-handler-ssl-ocsp-4.1.100.Final.jar":"System Classpath","/usr/lib/hadoop/hadoop-azure-datalake-3.4.1-amzn-0.jar":"System Classpath","/usr/lib/hudi/hudi-aws-bundle.jar":"System Classpath","/usr/lib/spark/jars/avro-1.11.2-cve-2024-47561-amzn-1.jar":"System Classpath","/usr/lib/spark/jars/spire-macros_2.12-0.17.0.jar":"System Classpath","/usr/share/java/Hive-JSON-Serde/json-1.3.8.jar":"System Classpath","/usr/lib/spark/jars/hadoop-client-api-3.4.1-amzn-0.jar":"System Classpath","/usr/share/aws/redshift/jdbc/RedshiftJDBC.jar":"System Classpath","/usr/share/aws/iceberg/lib/iceberg-flink-runtime-1.20-1.7.1-amzn-1.jar":"System Classpath","/usr/share/java/Hive-JSON-Serde/hive-openx-serde-1.3.8.jar":"System Classpath","/usr/lib/spark/jars/hadoop-yarn-server-web-proxy-3.4.1-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/commons-crypto-1.1.0.jar":"System Classpath","/usr/lib/hadoop/lib/zookeeper-jute-3.9.3-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/JLargeArrays-1.5.jar":"System Classpath","/usr/lib/spark/jars/shims-0.9.45.jar":"System Classpath","/usr/lib/hadoop/lib/commons-cli-1.5.0.jar":"System Classpath","/usr/lib/hadoop/lib/netty-transport-rxtx-4.1.100.Final.jar":"System Classpath","/usr/lib/hadoop/hadoop-common.jar":"System Classpath","/usr/lib/hadoop/lib/curator-recipes-5.2.0.jar":"System Classpath","/usr/lib/spark/jars/grpc-netty-1.56.0.jar":"System Classpath","/usr/lib/spark/jars/commons-codec-1.16.1.jar":"System Classpath","/usr/lib/spark/jars/hive-shims-2.3.9-amzn-4.jar":"System Classpath","/usr/lib/spark/jars/log4j-api-2.20.0.jar":"System Classpath","/usr/lib/spark/jars/antlr-runtime-3.5.2.jar":"System Classpath","/usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar":"System Classpath","/usr/share/aws/emr/s3select/lib/emr-s3-select-spark-connector.jar":"System Classpath","/usr/lib/spark/jars/spark-ganglia-lgpl_2.12-3.5.4-amzn-0.jar":"System Classpath","/usr/lib/hadoop/lib/netty-resolver-dns-native-macos-4.1.100.Final-osx-aarch_64.jar":"System Classpath","/usr/share/aws/emr/emrfs/auxlib/*":"System Classpath","/usr/lib/spark/jars/spark-catalyst_2.12-3.5.4-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/metrics-graphite-4.2.19.jar":"System Classpath","/usr/lib/spark/jars/avro-ipc-1.11.2-cve-2024-47561-amzn-1.jar":"System Classpath","/docker/usr/lib/hadoop/hadoop-aws.jar":"System Classpath","/usr/lib/hadoop/hadoop-nfs-3.4.1-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/spire-util_2.12-0.17.0.jar":"System Classpath","/usr/lib/spark/jars/log4j-core-2.20.0.jar":"System Classpath","/usr/lib/hadoop/lib/jaxb-impl-2.2.3-1.jar":"System Classpath","/docker/usr/share/aws/emr/emrfs/lib/*":"System Classpath","/usr/lib/hadoop/lib/netty-transport-classes-kqueue-4.1.100.Final.jar":"System Classpath","/usr/share/aws/emr/emrfs/lib/bcprov-ext-jdk15on-1.66.jar":"System Classpath","/usr/lib/hadoop/hadoop-archives.jar":"System Classpath","/docker/usr/share/aws/aws-java-sdk/*":"System Classpath","/usr/share/aws/delta/lib/delta-storage-s3-dynamodb-3.3.0-amzn-0.jar":"System Classpath","/usr/lib/hadoop/lib/netty-resolver-4.1.100.Final.jar":"System Classpath","/usr/share/java/Hive-JSON-Serde/json-serde-1.3.8.jar":"System Classpath","/usr/lib/hadoop/lib/snappy-java-1.1.10.4.jar":"System Classpath","/usr/lib/hadoop/hadoop-distcp.jar":"System Classpath","/usr/lib/spark/jars/commons-cli-1.5.0.jar":"System Classpath","/usr/lib/hadoop/lib/commons-configuration2-2.10.1.jar":"System Classpath","/usr/share/aws/emr/goodies/lib/emr-spark-goodies.jar":"System Classpath","/usr/share/aws/iceberg/lib/iceberg-spark-runtime-3.5_2.12-1.7.1-amzn-1.jar":"System Classpath","/usr/share/aws/redshift/jdbc/AwsSecretsManagerCachingJava.jar":"System Classpath","/usr/lib/hadoop/lib/jetty-servlet-9.4.53.v20231009.jar":"System Classpath","/usr/share/aws/emr/cloudwatch-sink/lib/*":"System Classpath","/usr/lib/spark/jars/datanucleus-api-jdo-4.2.4.jar":"System Classpath","/usr/lib/hadoop/lib/jersey-server-1.19.4.jar":"System Classpath","/usr/lib/hadoop/hadoop-federation-balance.jar":"System Classpath","/usr/lib/spark/jars/dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar":"System Classpath","/usr/lib/hadoop/hadoop-minicluster.jar":"System Classpath","/usr/lib/hadoop/lib/dnsjava-3.6.1.jar":"System Classpath","/usr/lib/spark/jars/jaxb-runtime-2.3.2.jar":"System Classpath","/usr/lib/spark/jars/hive-service-rpc-3.1.3.jar":"System Classpath","/usr/lib/spark/jars/grpc-api-1.56.0.jar":"System Classpath","/usr/lib/hadoop/hadoop-annotations-3.4.1-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/spark-network-shuffle_2.12-3.5.4-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/annotations-4.1.1.4.jar":"System Classpath","/usr/lib/hadoop-lzo/lib/hadoop-lzo.jar":"System Classpath","/usr/share/aws/redshift/jdbc/RedshiftJDBC42.jar":"System Classpath","/usr/lib/spark/jars/arpack_combined_all-0.1.jar":"System Classpath","/usr/share/aws/aws-java-sdk/aws-java-sdk-bundle-1.12.780.jar":"System Classpath","/tmp/glue-job-14058754211823993811/jars/OBgZGQ-AwsGlueMLLibs.jar":"System Classpath","/etc/hadoop/conf":"System Classpath","/usr/lib/hadoop/hadoop-kms.jar":"System Classpath","/usr/lib/spark/jars/spark-hive_2.12-3.5.4-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/compress-lzf-1.1.2.jar":"System Classpath","/usr/share/aws/glue-connectors/lib/GlueSparkConnector-MySQL-1.0.jar":"System Classpath","/usr/lib/hadoop/lib/slf4j-api-1.7.36.jar":"System Classpath","/usr/lib/hadoop/lib/netty-resolver-dns-native-macos-4.1.100.Final-osx-x86_64.jar":"System Classpath","/usr/lib/spark/jars/arrow-memory-core-12.0.1.jar":"System Classpath","/usr/share/aws/emr/emrfs/lib/objenesis-2.5.1.jar":"System Classpath","/usr/lib/spark/jars/spire-platform_2.12-0.17.0.jar":"System Classpath","/usr/lib/spark/jars/json4s-scalap_2.12-3.7.0-M11.jar":"System Classpath","/usr/lib/spark/jars/zookeeper-3.9.1.jar":"System Classpath","/usr/lib/spark/jars/commons-compress-1.23.0.jar":"System Classpath","/usr/lib/spark/jars/zstd-jni-1.5.5-4.jar":"System Classpath","/usr/lib/spark/jars/jakarta.validation-api-2.0.2.jar":"System Classpath","/usr/lib/spark/jars/hive-cli-2.3.9-amzn-4.jar":"System Classpath","/usr/share/aws/glue-connectors/lib/GlueSparkConnector-SAPHana-1.0.jar":"System Classpath","/usr/lib/spark/jars/grpc-services-1.56.0.jar":"System Classpath","/usr/lib/spark/jars/spark-kvstore_2.12-3.5.4-amzn-0.jar":"System Classpath","/usr/share/aws/emr/emrfs/lib/animal-sniffer-annotations-1.14.jar":"System Classpath","/usr/lib/hadoop/lib/commons-collections-3.2.2.jar":"System Classpath","/usr/lib/spark/jars/parquet-format-structures-1.13.1-amzn-6.jar":"System Classpath","/usr/lib/spark/jars/javassist-3.29.2-GA.jar":"System Classpath","/usr/lib/spark/jars/netty-handler-proxy-4.1.100.Final.jar":"System Classpath","/usr/lib/hadoop/lib/jersey-core-1.19.4.jar":"System Classpath","/usr/share/aws/glue-connectors/lib/GlueSparkConnector-AzureCosmos-1.0.jar":"System Classpath","/usr/lib/hadoop/hadoop-dynamometer-workload-3.4.1-amzn-0.jar":"System Classpath","/usr/lib/hadoop/lib/jersey-json-1.22.0.jar":"System Classpath","/usr/lib/spark/jars/threeten-extra-1.7.1.jar":"System Classpath","/usr/share/java/Hive-JSON-Serde/json-serde.jar":"System Classpath","/usr/lib/spark/jars/spark-core_2.12-3.5.4-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/blas-3.0.3.jar":"System Classpath","/usr/lib/hadoop/lib/kerby-config-2.0.3.jar":"System Classpath","/usr/lib/spark/jars/netty-transport-native-unix-common-4.1.100.Final.jar":"System Classpath","/usr/lib/spark/jars/json4s-core_2.12-3.7.0-M11.jar":"System Classpath","/docker/usr/share/aws/emr/security/lib/*":"System Classpath","/usr/lib/spark/jars/lz4-java-1.8.0.jar":"System Classpath","/usr/share/aws/emr/emrfs/lib/checker-qual-2.5.2.jar":"System Classpath","/usr/lib/spark/jars/super-csv-2.2.0.jar":"System Classpath","/usr/lib/spark/jars/jodd-core-3.5.2.jar":"System Classpath","/usr/share/aws/iceberg/lib/iceberg-hive3-runtime.jar":"System Classpath","/usr/lib/spark/jars/spark-streaming_2.12-3.5.4-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/grpc-context-1.56.0.jar":"System Classpath","/usr/lib/spark/jars/proto-google-common-protos-2.17.0.jar":"System Classpath","/usr/lib/hadoop/lib/kerb-util-2.0.3.jar":"System Classpath","/usr/lib/hadoop/hadoop-kms-3.4.1-amzn-0.jar":"System Classpath","/usr/lib/hadoop/hadoop-distcp-3.4.1-amzn-0.jar":"System Classpath","/usr/lib/hadoop/lib/curator-client-5.2.0.jar":"System Classpath","/usr/share/aws/emr/emrfs/conf":"System Classpath","/usr/lib/hadoop/lib/jackson-databind-2.12.7.1.jar":"System Classpath","/usr/share/aws/glue-connectors/lib/GlueSparkConnector-Teradata-1.0.jar":"System Classpath","/usr/lib/hadoop/lib/commons-codec-1.15.jar":"System Classpath","/usr/share/aws/glue-connectors/lib/GlueSparkConnector-Oracle-1.0.jar":"System Classpath","/usr/lib/hadoop/lib/kerb-core-2.0.3.jar":"System Classpath","/usr/lib/spark/jars/hk2-api-2.6.1.jar":"System Classpath","/usr/lib/hadoop/hadoop-aws.jar":"System Classpath","/usr/share/aws/hmclient/lib/aws-glue-datacatalog-hive3-client.jar":"System Classpath","/usr/lib/spark/jars/hive-jdbc-2.3.9-amzn-4.jar":"System Classpath","/usr/lib/spark/jars/jakarta.ws.rs-api-2.1.6.jar":"System Classpath","/usr/share/aws/glue-connectors/lib/GlueSparkConnector-Snowflake-1.0.jar":"System Classpath","/usr/lib/spark/jars/netty-transport-classes-kqueue-4.1.100.Final.jar":"System Classpath","/usr/share/aws/glue-connectors/lib/GlueSparkConnector-MongoDB-1.0.jar":"System Classpath","/usr/lib/hadoop/hadoop-archives-3.4.1-amzn-0.jar":"System Classpath","/usr/share/aws/delta/lib/delta-standalone-original-shaded_2.12-3.3.0-amzn-0.jar":"System Classpath","/usr/lib/hadoop/hadoop-client-3.4.1-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/ST4-4.0.4.jar":"System Classpath","/usr/lib/hadoop/hadoop-extras.jar":"System Classpath","/usr/lib/spark/jars/opencsv-2.3.jar":"System Classpath","/usr/lib/spark/jars/stax-api-1.0.1.jar":"System Classpath","/usr/lib/hadoop/lib/netty-transport-udt-4.1.100.Final.jar":"System Classpath","/usr/lib/spark/jars/slf4j-api-2.0.7.jar":"System Classpath","/usr/lib/spark/jars/javolution-5.5.1.jar":"System Classpath","/usr/lib/spark/jars/jul-to-slf4j-2.0.7.jar":"System Classpath","/usr/share/aws/redshift/spark-redshift/lib/spark-avro.jar":"System Classpath","/usr/share/aws/hmclient/lib/aws-glue-datacatalog-client-common-4.5.0.jar":"System Classpath","/usr/lib/spark/jars/logging-interceptor-3.12.12.jar":"System Classpath","/usr/lib/spark/jars/parquet-encoding-1.13.1-amzn-6.jar":"System Classpath","/usr/share/aws/emr/emrfs/lib/annotations-16.0.2.jar":"System Classpath","/usr/lib/hadoop/hadoop-minicluster-3.4.1-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/scala-library-2.12.18.jar":"System Classpath","/docker/usr/share/aws/hmclient/lib/aws-glue-datacatalog-spark-client.jar":"System Classpath","/usr/lib/spark/jars/metrics-json-4.2.19.jar":"System Classpath","/usr/lib/hadoop/lib/commons-math3-3.6.1.jar":"System Classpath","/usr/lib/spark/jars/guava-14.0.1.jar":"System Classpath","/usr/share/aws/emr/emrfs/lib/j2objc-annotations-1.1.jar":"System Classpath","/usr/share/aws/emr/emrfs/lib/aopalliance-1.0.jar":"System Classpath","/usr/lib/hadoop/lib/jaxb-api-2.2.11.jar":"System Classpath","/usr/share/aws/delta/lib/delta-iceberg_2.12-3.3.0-amzn-0.jar":"System Classpath","/usr/lib/hadoop/lib/commons-lang3-3.12.0.jar":"System Classpath","/usr/lib/spark/jars/lapack-3.0.3.jar":"System Classpath","/usr/lib/hadoop/lib/jetty-util-ajax-9.4.53.v20231009.jar":"System Classpath","/usr/lib/hadoop/lib/commons-daemon-1.0.13.jar":"System Classpath","/usr/lib/hadoop/lib/netty-codec-smtp-4.1.100.Final.jar":"System Classpath","/usr/lib/hadoop/lib/netty-handler-proxy-4.1.100.Final.jar":"System Classpath","/usr/lib/spark/conf":"System Classpath","/usr/lib/hadoop/lib/netty-transport-native-kqueue-4.1.100.Final-osx-aarch_64.jar":"System Classpath","/usr/lib/hadoop/lib/jetty-server-9.4.53.v20231009.jar":"System Classpath","/usr/lib/spark/jars/spark-yarn_2.12-3.5.4-amzn-0.jar":"System Classpath","/usr/lib/spark/jars/jakarta.xml.bind-api-2.3.2.jar":"System Classpath","/usr/lib/hadoop/lib/netty-handler-4.1.100.Final.jar":"System Classpath","/usr/lib/hadoop/hadoop-azure-3.4.1-amzn-0.jar":"System Classpath","/usr/lib/hadoop/lib/netty-buffer-4.1.100.Final.jar":"System Classpath","/docker/usr/lib/hadoop-lzo/lib/*":"System Classpath"}}
{"Event":"SparkListenerApplicationStart","App Name":"nativespark-test_zodiac_pg_to_redshift-jr_4464f784f631b1a0c7231da55ee1a6f368615377dff170f59beab638b8587351","App ID":"spark-application-1761565571107","Timestamp":1761565568831,"User":"hadoop","Driver Logs":{},"Driver Attributes":{}}
{"Event":"SparkListenerExecutorAdded","Timestamp":1761565580593,"Executor ID":"4","Executor Info":{"Host":"172.36.35.56","Total Cores":4,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0,"Registration Time":1761565580592}}
{"Event":"SparkListenerExecutorAdded","Timestamp":1761565580594,"Executor ID":"3","Executor Info":{"Host":"172.36.138.186","Total Cores":4,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0,"Registration Time":1761565580594}}
{"Event":"SparkListenerExecutorAdded","Timestamp":1761565580631,"Executor ID":"1","Executor Info":{"Host":"172.36.64.223","Total Cores":4,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0,"Registration Time":1761565580630}}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"4","Host":"172.36.35.56","Port":40473},"Maximum Memory":6253707264,"Timestamp":1761565580688,"Maximum Onheap Memory":6253707264,"Maximum Offheap Memory":0}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"3","Host":"172.36.138.186","Port":46711},"Maximum Memory":6253707264,"Timestamp":1761565580699,"Maximum Onheap Memory":6253707264,"Maximum Offheap Memory":0}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"1","Host":"172.36.64.223","Port":33687},"Maximum Memory":6253707264,"Timestamp":1761565580707,"Maximum Onheap Memory":6253707264,"Maximum Offheap Memory":0}
{"Event":"SparkListenerExecutorAdded","Timestamp":1761565581168,"Executor ID":"6","Executor Info":{"Host":"172.34.231.178","Total Cores":4,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0,"Registration Time":1761565581168}}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"6","Host":"172.34.231.178","Port":41563},"Maximum Memory":6253707264,"Timestamp":1761565581230,"Maximum Onheap Memory":6253707264,"Maximum Offheap Memory":0}
{"Event":"SparkListenerExecutorAdded","Timestamp":1761565581254,"Executor ID":"8","Executor Info":{"Host":"172.36.104.93","Total Cores":4,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0,"Registration Time":1761565581254}}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"8","Host":"172.36.104.93","Port":35597},"Maximum Memory":6253707264,"Timestamp":1761565581340,"Maximum Onheap Memory":6253707264,"Maximum Offheap Memory":0}
{"Event":"SparkListenerExecutorAdded","Timestamp":1761565581535,"Executor ID":"5","Executor Info":{"Host":"172.34.237.233","Total Cores":4,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0,"Registration Time":1761565581535}}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"5","Host":"172.34.237.233","Port":39131},"Maximum Memory":6253707264,"Timestamp":1761565581617,"Maximum Onheap Memory":6253707264,"Maximum Offheap Memory":0}
{"Event":"SparkListenerExecutorAdded","Timestamp":1761565582090,"Executor ID":"7","Executor Info":{"Host":"172.36.63.85","Total Cores":4,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0,"Registration Time":1761565582090}}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"7","Host":"172.36.63.85","Port":35845},"Maximum Memory":6253707264,"Timestamp":1761565582152,"Maximum Onheap Memory":6253707264,"Maximum Offheap Memory":0}
{"Event":"SparkListenerExecutorAdded","Timestamp":1761565586538,"Executor ID":"2","Executor Info":{"Host":"172.36.80.60","Total Cores":4,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0,"Registration Time":1761565586538}}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"2","Host":"172.36.80.60","Port":35855},"Maximum Memory":6253707264,"Timestamp":1761565586600,"Maximum Onheap Memory":6253707264,"Maximum Offheap Memory":0}
{"Event":"SparkListenerExecutorAdded","Timestamp":1761565589813,"Executor ID":"9","Executor Info":{"Host":"172.34.155.211","Total Cores":4,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0,"Registration Time":1761565589813}}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"9","Host":"172.34.155.211","Port":40325},"Maximum Memory":6253707264,"Timestamp":1761565589901,"Maximum Onheap Memory":6253707264,"Maximum Offheap Memory":0}