Skip to content

Commit

Permalink
feat: add opt-in support for S3 us-east-1 regional endpoint (#2960)
Browse files Browse the repository at this point in the history
* generalize resolveRegionalEndpointsFlag function; update STS customization
  • Loading branch information
AllanZhengYP authored Nov 27, 2019
1 parent 151ca09 commit 2f3826c
Show file tree
Hide file tree
Showing 8 changed files with 328 additions and 98 deletions.
5 changes: 5 additions & 0 deletions .changes/next-release/feature-Region-e1805cab.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"type": "feature",
"category": "Region",
"description": "s3 client now support sending request to us-east-1 regional endpoint with `s3UsEast1RegionalEndpoint` client configuration set to `regional`"
}
13 changes: 12 additions & 1 deletion lib/config.js
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,12 @@ var PromisesDependency;
* @return [Boolean] whether to disable S3 body signing when using signature version `v4`.
* Body signing can only be disabled when using https. Defaults to `true`.
*
* @!attribute s3UsEast1RegionalEndpoint
* @return ['legacy'|'regional'] when region is set to 'us-east-1', whether to send s3
* request to global endpoints or 'us-east-1' regional endpoints. This config is only
* applicable to S3 client;
* Defaults to 'legacy'
*
* @!attribute useAccelerateEndpoint
* @note This configuration option is only compatible with S3 while accessing
* dns-compatible buckets.
Expand Down Expand Up @@ -238,6 +244,10 @@ AWS.Config = AWS.util.inherit({
* @option options s3DisableBodySigning [Boolean] whether S3 body signing
* should be disabled when using signature version `v4`. Body signing
* can only be disabled when using https. Defaults to `true`.
* @option options s3UsEast1RegionalEndpoint ['legacy'|'regional'] when region
* is set to 'us-east-1', whether to send s3 request to global endpoints or
* 'us-east-1' regional endpoints. This config is only applicable to S3 client.
* Defaults to `legacy`
*
* @option options retryDelayOptions [map] A set of options to configure
* the retry delay on retryable errors. Currently supported options are:
Expand Down Expand Up @@ -523,6 +533,7 @@ AWS.Config = AWS.util.inherit({
s3ForcePathStyle: false,
s3BucketEndpoint: false,
s3DisableBodySigning: true,
s3UsEast1RegionalEndpoint: 'legacy',
computeChecksums: true,
convertResponseTypes: true,
correctClockSkew: false,
Expand All @@ -537,7 +548,7 @@ AWS.Config = AWS.util.inherit({
endpointDiscoveryEnabled: false,
endpointCacheSize: 1000,
hostPrefixEnabled: true,
stsRegionalEndpoints: null
stsRegionalEndpoints: 'legacy'
},

/**
Expand Down
67 changes: 67 additions & 0 deletions lib/config_regional_endpoint.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
var AWS = require('./core');
/**
* @api private
*/
function validateRegionalEndpointsFlagValue(configValue, errorOptions) {
if (typeof configValue !== 'string') return undefined;
else if (['legacy', 'regional'].indexOf(configValue.toLowerCase()) >= 0) {
return configValue.toLowerCase();
} else {
throw AWS.util.error(new Error(), errorOptions);
}
}

/**
* Resolve the configuration value for regional endpoint from difference sources: client
* config, environmental variable, shared config file. Value can be case-insensitive
* 'legacy' or 'reginal'.
* @param originalConfig user-supplied config object to resolve
* @param options a map of config property names from individual configuration source
* - env: name of environmental variable that refers to the config
* - sharedConfig: name of shared configuration file property that refers to the config
* - clientConfig: name of client configuration property that refers to the config
*
* @api private
*/
function resolveRegionalEndpointsFlag(originalConfig, options) {
originalConfig = originalConfig || {};
//validate config value
var resolved;
if (originalConfig[options.clientConfig]) {
resolved = validateRegionalEndpointsFlagValue(originalConfig[options.clientConfig], {
code: 'InvalidConfiguration',
message: 'invalid "' + options.clientConfig + '" configuration. Expect "legacy" ' +
' or "regional". Got "' + originalConfig[options.clientConfig] + '".'
});
if (resolved) return resolved;
}
if (!AWS.util.isNode()) return resolved;
//validate environmental variable
if (Object.prototype.hasOwnProperty.call(process.env, options.env)) {
var envFlag = process.env[options.env];
resolved = validateRegionalEndpointsFlagValue(envFlag, {
code: 'InvalidEnvironmentalVariable',
message: 'invalid ' + options.env + ' environmental variable. Expect "legacy" ' +
' or "regional". Got "' + process.env[options.env] + '".'
});
if (resolved) return resolved;
}
//validate shared config file
var profile = {};
try {
var profiles = AWS.util.getProfilesFromSharedConfig(AWS.util.iniLoader);
profile = profiles[process.env.AWS_PROFILE || AWS.util.defaultProfile];
} catch (e) {};
if (profile && Object.prototype.hasOwnProperty.call(profile, options.sharedConfig)) {
var fileFlag = profile[options.sharedConfig];
resolved = validateRegionalEndpointsFlagValue(fileFlag, {
code: 'InvalidConfiguration',
message: 'invalid ' + options.sharedConfig + ' profile config. Expect "legacy" ' +
' or "regional". Got "' + profile[options.sharedConfig] + '".'
});
if (resolved) return resolved;
}
return resolved;
}

module.exports = resolveRegionalEndpointsFlag;
30 changes: 30 additions & 0 deletions lib/services/s3.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
var AWS = require('../core');
var v4Credentials = require('../signers/v4_credentials');
var resolveRegionalEndpointsFlag = require('../config_regional_endpoint');

// Pull in managed upload extension
require('../s3/managed_upload');
Expand Down Expand Up @@ -105,6 +106,7 @@ AWS.util.update(AWS.S3.prototype, {
request.addListener('validate', this.validateBucketEndpoint);
request.addListener('validate', this.correctBucketRegionFromCache);
request.addListener('validate', this.validateBucketName, prependListener);
request.addListener('validate', this.optInUsEast1RegionalEndpoint, prependListener);

request.addListener('build', this.addContentType);
request.addListener('build', this.populateURI);
Expand Down Expand Up @@ -189,6 +191,34 @@ AWS.util.update(AWS.S3.prototype, {
return invalidOperations.indexOf(operation) === -1;
},

/**
* When us-east-1 region endpoint configuration is set, in stead of sending request to
* global endpoint(e.g. 's3.amazonaws.com'), we will send request to
* 's3.us-east-1.amazonaws.com'.
* @api private
*/
optInUsEast1RegionalEndpoint: function optInUsEast1RegionalEndpoint(req) {
var service = req.service;
var config = service.config;
config.s3UsEast1RegionalEndpoint = resolveRegionalEndpointsFlag(service._originalConfig, {
env: 'AWS_S3_US_EAST_1_REGIONAL_ENDPOINT',
sharedConfig: 's3_us_east_1_regional_endpoint',
clientConfig: 's3UsEast1RegionalEndpoint'
});
if (
!(service._originalConfig || {}).endpoint &&
req.httpRequest.region === 'us-east-1' &&
config.s3UsEast1RegionalEndpoint === 'regional' &&
req.httpRequest.endpoint.hostname.indexOf('s3.amazonaws.com') >= 0
) {
var insertPoint = config.endpoint.indexOf('.amazonaws.com');
regionalEndpoint = config.endpoint.substring(0, insertPoint) +
'.us-east-1' + config.endpoint.substring(insertPoint);
var endpoint = req.httpRequest.endpoint;
endpoint.hostname = regionalEndpoint;
endpoint.host = regionalEndpoint;
}
},

/**
* S3 prefers dns-compatible bucket names to be moved from the uri path
Expand Down
79 changes: 20 additions & 59 deletions lib/services/sts.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
var AWS = require('../core');
var regionConfig = require('../region_config');
var resolveRegionalEndpointsFlag = require('../config_regional_endpoint');
var ENV_REGIONAL_ENDPOINT_ENABLED = 'AWS_STS_REGIONAL_ENDPOINTS';
var CONFIG_REGIONAL_ENDPOINT_ENABLED = 'sts_regional_endpoints';

Expand Down Expand Up @@ -51,77 +51,38 @@ AWS.util.update(AWS.STS.prototype, {
/**
* @api private
*/
validateRegionalEndpointsFlagValue: function validateRegionalEndpointsFlagValue(configValue, errorOptions) {
if (typeof configValue === 'string' && ['legacy', 'regional'].indexOf(configValue.toLowerCase()) >= 0) {
this.config.stsRegionalEndpoints = configValue.toLowerCase();
return;
} else {
throw AWS.util.error(new Error(), errorOptions);
}
setupRequestListeners: function setupRequestListeners(request) {
request.addListener('validate', this.optInRegionalEndpoint, true);
},

/**
* @api private
*/
validateRegionalEndpointsFlag: function validateRegionalEndpointsFlag() {
//validate config value
var config = this.config;
if (config.stsRegionalEndpoints) {
this.validateRegionalEndpointsFlagValue(config.stsRegionalEndpoints, {
code: 'InvalidConfiguration',
message: 'invalid "stsRegionalEndpoints" configuration. Expect "legacy" ' +
' or "regional". Got "' + config.stsRegionalEndpoints + '".'
});
}
if (!AWS.util.isNode()) return;
//validate environmental variable
if (Object.prototype.hasOwnProperty.call(process.env, ENV_REGIONAL_ENDPOINT_ENABLED)) {
var envFlag = process.env[ENV_REGIONAL_ENDPOINT_ENABLED];
this.validateRegionalEndpointsFlagValue(envFlag, {
code: 'InvalidEnvironmentalVariable',
message: 'invalid ' + ENV_REGIONAL_ENDPOINT_ENABLED + ' environmental variable. Expect "legacy" ' +
' or "regional". Got "' + process.env[ENV_REGIONAL_ENDPOINT_ENABLED] + '".'
});
}
//validate shared config file
var profile = {};
try {
var profiles = AWS.util.getProfilesFromSharedConfig(AWS.util.iniLoader);
profile = profiles[process.env.AWS_PROFILE || AWS.util.defaultProfile];
} catch (e) {};
if (profile && Object.prototype.hasOwnProperty.call(profile, CONFIG_REGIONAL_ENDPOINT_ENABLED)) {
var fileFlag = profile[CONFIG_REGIONAL_ENDPOINT_ENABLED];
this.validateRegionalEndpointsFlagValue(fileFlag, {
code: 'InvalidConfiguration',
message: 'invalid '+CONFIG_REGIONAL_ENDPOINT_ENABLED+' profile config. Expect "legacy" ' +
' or "regional". Got "' + profile[CONFIG_REGIONAL_ENDPOINT_ENABLED] + '".'
});
}
},

/**
* @api private
*/
optInRegionalEndpoint: function optInRegionalEndpoint() {
this.validateRegionalEndpointsFlag();
var config = this.config;
if (config.stsRegionalEndpoints === 'regional') {
regionConfig(this);
if (!this.isGlobalEndpoint) return;
this.isGlobalEndpoint = false;
optInRegionalEndpoint: function optInRegionalEndpoint(req) {
var service = req.service;
var config = service.config;
config.stsRegionalEndpoints = resolveRegionalEndpointsFlag(service._originalConfig, {
env: ENV_REGIONAL_ENDPOINT_ENABLED,
sharedConfig: CONFIG_REGIONAL_ENDPOINT_ENABLED,
clientConfig: 'stsRegionalEndpoints'
});
if (
config.stsRegionalEndpoints === 'regional' &&
service.isGlobalEndpoint
) {
//client will throw if region is not supplied; request will be signed with specified region
if (!config.region) {
throw AWS.util.error(new Error(),
{code: 'ConfigError', message: 'Missing region in config'});
}
var insertPoint = config.endpoint.indexOf('.amazonaws.com');
config.endpoint = config.endpoint.substring(0, insertPoint) +
regionalEndpoint = config.endpoint.substring(0, insertPoint) +
'.' + config.region + config.endpoint.substring(insertPoint);
var endpoint = req.httpRequest.endpoint;
endpoint.hostname = regionalEndpoint;
endpoint.host = regionalEndpoint;
req.httpRequest.region = config.region;
}
},

validateService: function validateService() {
this.optInRegionalEndpoint();
}

});
28 changes: 18 additions & 10 deletions scripts/region-checker/whitelist.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,11 @@ var whitelist = {
'/config.js': [
24,
25,
187
85,
86,
193,
247,
248
],
'/credentials/cognito_identity_credentials.js': [
78,
Expand All @@ -25,16 +29,20 @@ var whitelist = {
316
],
'/services/s3.js': [
68,
69,
515,
517,
516,
636,
647,
648,
649,
654
70,
194,
196,
209,
215,
545,
546,
547,
666,
677,
678,
684,
679
]
};

Expand Down
Loading

0 comments on commit 2f3826c

Please sign in to comment.