-
Notifications
You must be signed in to change notification settings - Fork 877
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
c99b370
commit 4925505
Showing
13 changed files
with
421 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
name: aws-ts-apigateway | ||
runtime: nodejs | ||
description: Basic example of using AWS API Gateway | ||
template: | ||
description: Basic example of using AWS API Gateway | ||
config: | ||
aws:region: | ||
description: The AWS region to deploy into | ||
default: us-east-2 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,77 @@ | ||
[![Deploy](https://get.pulumi.com/new/button.svg)](https://app.pulumi.com/new) | ||
|
||
# Serverless REST API on AWS | ||
|
||
A simple REST API that counts the number of times a route has been hit. For a detailed walkthrough of this example, see the article [Create a Serverless REST API](https://pulumi.io/quickstart/aws-rest-api.html). | ||
|
||
## Deploying and running the program | ||
|
||
Note: some values in this example will be different from run to run. These values are indicated | ||
with `***`. | ||
|
||
1. Create a new stack: | ||
|
||
```bash | ||
$ pulumi stack init count-api-testing | ||
``` | ||
|
||
1. Set the AWS region: | ||
|
||
``` | ||
$ pulumi config set aws:region us-east-2 | ||
``` | ||
|
||
1. Restore NPM modules via `npm install` or `yarn install`. | ||
|
||
1. Run `pulumi up` to preview and deploy changes: | ||
|
||
``` | ||
$ pulumi up | ||
Previewing update of stack 'count-api-testing' | ||
... | ||
|
||
Updating (count-api-testing): | ||
|
||
Type Name Status | ||
+ pulumi:pulumi:Stack aws-ts-apigateway-count-api-testing created | ||
+ ├─ aws:apigateway:x:API hello-world created | ||
+ │ ├─ aws:iam:Role hello-world4fcc7b60 created | ||
+ │ ├─ aws:iam:RolePolicyAttachment hello-world4fcc7b60-32be53a2 created | ||
+ │ ├─ aws:lambda:Function hello-world4fcc7b60 created | ||
+ │ ├─ aws:apigateway:RestApi hello-world created | ||
+ │ ├─ aws:apigateway:Deployment hello-world created | ||
+ │ ├─ aws:lambda:Permission hello-world-a552609d created | ||
+ │ └─ aws:apigateway:Stage hello-world created | ||
+ └─ aws:dynamodb:Table counterTable created | ||
|
||
Outputs: | ||
endpoint: "https://***execute-api.us-east-2.amazonaws.com/stage/" | ||
|
||
Resources: | ||
+ 10 created | ||
|
||
Duration: 24s | ||
|
||
1. View the endpoint URL and curl a few routes: | ||
|
||
```bash | ||
$ pulumi stack output | ||
Current stack outputs (1): | ||
OUTPUT VALUE | ||
endpoint https://***.us-east-2.amazonaws.com/stage/ | ||
$ curl $(pulumi stack output endpoint)/hello | ||
{"route":"hello","count":1} | ||
$ curl $(pulumi stack output endpoint)/hello | ||
{"route":"hello","count":2} | ||
$ curl $(pulumi stack output endpoint)/woohoo | ||
{"route":"woohoo","count":1} | ||
``` | ||
|
||
1. To view the runtime logs of the Lambda function, use the `pulumi logs` command. To get a log stream, use `pulumi logs --follow`. | ||
|
||
## Clean up | ||
|
||
1. Run `pulumi destroy` to tear down all resources. | ||
|
||
1. To delete the stack itself, run `pulumi stack rm`. Note that this command deletes all deployment history from the Pulumi Console. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,50 @@ | ||
import * as aws from "@pulumi/aws"; | ||
|
||
// Create a mapping from 'route' to a count | ||
let counterTable = new aws.dynamodb.Table("counterTable", { | ||
attributes: [{ | ||
name: "id", | ||
type: "S", | ||
}], | ||
hashKey: "id", | ||
readCapacity: 5, | ||
writeCapacity: 5, | ||
}); | ||
|
||
// Create an API endpoint | ||
let endpoint = new aws.apigateway.x.API("hello-world", { | ||
routes: [{ | ||
path: "/{route+}", | ||
method: "GET", | ||
eventHandler: async (event) => { | ||
let route = event.pathParameters["route"]; | ||
console.log(`Getting count for '${route}'`); | ||
|
||
const client = new aws.sdk.DynamoDB.DocumentClient(); | ||
|
||
// get previous value and increment | ||
// reference outer `counterTable` object | ||
const tableData = await client.get({ | ||
TableName: counterTable.name.get(), | ||
Key: { id: route }, | ||
ConsistentRead: true, | ||
}).promise(); | ||
|
||
const value = tableData.Item; | ||
let count = (value && value.count) || 0; | ||
|
||
await client.put({ | ||
TableName: counterTable.name.get(), | ||
Item: { id: route, count: ++count }, | ||
}).promise(); | ||
|
||
console.log(`Got count ${count} for '${route}'`); | ||
return { | ||
statusCode: 200, | ||
body: JSON.stringify({ route, count }), | ||
}; | ||
}, | ||
}], | ||
}); | ||
|
||
exports.endpoint = endpoint.url; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
{ | ||
"name": "aws-ts-apigateway", | ||
"version": "0.1.0", | ||
"dependencies": { | ||
"@pulumi/aws": "dev", | ||
"@pulumi/pulumi": "dev" | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
{ | ||
"compilerOptions": { | ||
"outDir": "bin", | ||
"target": "es6", | ||
"module": "commonjs", | ||
"moduleResolution": "node", | ||
"sourceMap": true, | ||
"experimentalDecorators": true, | ||
"pretty": true, | ||
"noFallthroughCasesInSwitch": true, | ||
"noImplicitAny": true, | ||
"noImplicitReturns": true, | ||
"forceConsistentCasingInFileNames": true, | ||
"strictNullChecks": true | ||
}, | ||
"files": [ | ||
"index.ts" | ||
] | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
name: aws-ts-twitter-athena | ||
description: Analyze tweets in AWS Athena | ||
runtime: nodejs | ||
template: | ||
description: Analyze tweets in AWS Athena | ||
config: | ||
aws:region: | ||
description: The AWS region to deploy into | ||
default: us-west-2 | ||
twitterAccessTokenKey: | ||
description: Access token key | ||
twitterAccessTokenSecret: | ||
description: Access token secret | ||
secret: true | ||
twitterConsumerKey: | ||
description: Consumer key | ||
twitterConsumerSecret: | ||
description: Consumer secret | ||
secret: true | ||
twitterQuery: | ||
description: Search term to query for | ||
default: Amazon Web Services |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,67 @@ | ||
[![Deploy](https://get.pulumi.com/new/button.svg)](https://app.pulumi.com/new) | ||
|
||
# Twitter Search in Athena | ||
|
||
A sample project that queries Twitter every 2 minutes and stores the results in S3. The project also sets up an Athena table and query. This project demonstrates using `aws.cloudwatch.EventRule` to run a Lambda on an interval. | ||
|
||
## Setup | ||
|
||
Register a new [Twitter app](https://apps.twitter.com/). | ||
|
||
## Deploy and run the program | ||
|
||
1. Create a new stack: | ||
|
||
``` | ||
pulumi stack init twitter-athena | ||
``` | ||
|
||
1. In Twitter, get the keys for your application. Set configuration values for your Twitter consumer key/secret and application key/secret. Use the `--secret` flag to securely encrypt secret values. | ||
|
||
``` | ||
pulumi config set twitterAccessTokenKey <Value for Consumer Key (API Key)> | ||
pulumi config set --secret twitterAccessTokenSecret <Value for Consumer Secret (API Secret)> | ||
pulumi config set twitterConsumerKey <Value for Access Token> | ||
pulumi config set --secret twitterConsumerSecret <Value for Access Token Secret> | ||
``` | ||
|
||
1. Set a search term to query for: | ||
|
||
``` | ||
pulumi config set twitterQuery "Amazon Web Services" | ||
``` | ||
|
||
1. Set the AWS region: | ||
|
||
```bash | ||
pulumi config set aws:region us-west-2 | ||
``` | ||
|
||
1. Restore NPM modules via `npm install`. | ||
|
||
1. Preview and run the deployment via `pulumi up`. A total of 16 resources are created. | ||
|
||
1. Run `pulumi stack output` to view output properties (or view the stack on pulumi.com). | ||
|
||
``` | ||
$ pulumi stack output | ||
Please choose a stack: aws-serverless-js-twitter-dev | ||
Current stack outputs (4): | ||
OUTPUT VALUE | ||
athenaDatabase tweets_database | ||
bucketName tweet-bucket-de18828 | ||
createTableQueryUri https://us-west-2.console.aws.amazon.com/athena/home?force#query/saved/e394800e-a35e-44b3-b8ca-8b47b0f74469 | ||
topUsersQueryUri https://us-west-2.console.aws.amazon.com/athena/home?force#query/saved/51fa5744-bab6-4e5f-8cd6-9447b6619f06 | ||
``` | ||
|
||
1. Navigate to the URL for `createTableQueryUri` and run the query in the Athena console. This will create a table called `tweets`. | ||
|
||
1. Navigate to the URL for `topUsersQueryUri` and run the query in Athena. You'll see tweets for your search term, by users with more than 1000 followers. | ||
![Athena console](athena-screenshot.png) | ||
## Clean up | ||
To clean up resources, run `pulumi destroy` and answer the confirmation question at the prompt. | ||
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,123 @@ | ||
import * as pulumi from "@pulumi/pulumi"; | ||
import * as aws from "@pulumi/aws"; | ||
|
||
const bucket = new aws.s3.Bucket("tweet-bucket", { | ||
serverSideEncryptionConfiguration: { | ||
rule: { | ||
applyServerSideEncryptionByDefault: { | ||
sseAlgorithm: "AES256", | ||
}, | ||
}, | ||
}, | ||
}); | ||
let bucketName = bucket.id; | ||
|
||
let config = new pulumi.Config(); | ||
let consumerKey = config.require("twitterConsumerKey"); | ||
let consumerSecret = config.require("twitterConsumerSecret"); | ||
let accessTokenKey = config.require("twitterAccessTokenKey"); | ||
let accessTokenSecret = config.require("twitterAccessTokenSecret"); | ||
|
||
let twitterQuery = config.require("twitterQuery"); | ||
const outputFolder = "tweets"; | ||
|
||
let eventRule = new aws.cloudwatch.EventRule("twitter-search-timer", { | ||
scheduleExpression: "rate(1 minute)" | ||
}); | ||
|
||
let handler = eventRule.onEvent("on-timer-event", async() => { | ||
console.log("Timer fired."); | ||
let twitterClient = require("twitter"); | ||
var client = new twitterClient({ | ||
consumer_key: consumerKey, | ||
consumer_secret: consumerSecret, | ||
access_token_key: accessTokenKey, | ||
access_token_secret: accessTokenSecret, | ||
}); | ||
|
||
const tweets = await new Promise<string[]>((resolve, reject) => { | ||
client.get('search/tweets', {q: twitterQuery, count: 100}, function(error: any, tweets: any, response: any) { | ||
if (error) { | ||
return reject(error); | ||
} | ||
|
||
let statuses = tweets.statuses; | ||
console.log(`Got ${statuses.length} statuses.`); | ||
|
||
let results = statuses.map((s: any) => { | ||
let user = s.user.screen_name; | ||
|
||
return JSON.stringify({ | ||
created_at: s.created_at, | ||
id: s.id_str, | ||
text: s.text, | ||
user: user, | ||
hashtags: s.entities.hashtags, | ||
followers: s.user.followers_count, | ||
isVerified: s.user.verified, | ||
isRetweet: s.retweeted_status != null, | ||
url: `https://twitter.com/${user}/status/${s.id_str}`, | ||
}); | ||
}); | ||
|
||
return resolve(results); | ||
}); | ||
}); | ||
|
||
console.log(`Got ${tweets.length} tweets from Twitter for query ${twitterQuery}`); | ||
|
||
let filename = `${outputFolder}/${Date.now()}`; | ||
let contents = Buffer.from(tweets.join("\n"), "utf8"); | ||
|
||
let s3 = new aws.sdk.S3(); | ||
await s3.putObject({ | ||
Bucket: bucket.id.get(), | ||
Key: filename, | ||
Body: contents, | ||
}).promise(); | ||
}); | ||
|
||
// athena setup | ||
let athena = new aws.athena.Database("tweets_database", | ||
{ name: "tweets_database", bucket: bucket.id, forceDestroy: true } | ||
); | ||
|
||
// Sadly, there isn't support for Athena tables in Terraform. | ||
// See https://github.com/terraform-providers/terraform-provider-aws/pull/1893#issuecomment-351300973 | ||
// So, we'll instead create a query for the table definition | ||
function createTableQuery(bucket: string) { | ||
return `CREATE EXTERNAL TABLE IF NOT EXISTS tweets ( | ||
id string, | ||
text string, | ||
user string, | ||
isVerified boolean, | ||
url string, | ||
followers int, | ||
hashtags string, | ||
isRetweet boolean | ||
) | ||
ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe' | ||
LOCATION 's3://${bucket}/${outputFolder}/';`; | ||
} | ||
|
||
let topUsersQuery = | ||
`select distinct user, followers, text, url | ||
from tweets | ||
where isRetweet = false and followers > 1000 | ||
order by followers desc`; | ||
|
||
let createTableAthenaQuery = new aws.athena.NamedQuery( | ||
"createTable", { database: athena.id, query: bucketName.apply(createTableQuery)}); | ||
|
||
let topUsersAthenaQuery = new aws.athena.NamedQuery("topUsers", { database: athena.id, query: topUsersQuery}); | ||
|
||
function getQueryUri(queryId: string) { | ||
let config = new pulumi.Config("aws"); | ||
let region = config.require("region"); | ||
return `https://${region}.console.aws.amazon.com/athena/home?force#query/saved/${queryId}`; | ||
} | ||
|
||
exports.bucketName = bucketName | ||
exports.athenaDatabase = athena.id; | ||
exports.topUsersQueryUri = topUsersAthenaQuery.id.apply(getQueryUri); | ||
exports.createTableQueryUri = createTableAthenaQuery.id.apply(getQueryUri); |
Oops, something went wrong.
This is unfortunate. Notably the implementation of
getLinuxAmi
here doesn't even use this argument (!) - but also it's a shame that adding members to enums will cause this kind of break in any components which accept the enum...