-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
914 [Feature] - "Add remaining CRUD endpoints to SST" (#943)
* Implemented skeleton; still need to fill in handler implementation * Work in progress * Added type hints * Implemented get_trainspace endpoint * Progress * Progress * Resolved ticket 914 * Implemented DynamoDB built-in pagination consideration to get all trainspace endpoint * Installed yarn uuid package * Installed dependencies to serverless directory * Added types/uuid * Implemented PR feedback * T * Fixed stuff * Implemented PR changes: Narrowewd scope of SST handler permissions, cleaned up code, tested * 🎨 Auto-generated directory tree for repository in Architecture.md * Removed useless comment * Split creation of different trainspaces into different API endpoints * Implemented Daniel Wu's PR comments * Fixed SonarCloud comment changes * Renamed file * Fixed path issues * FF * Implemented Daniel Wu's PR comments * 🎨 Auto-generated directory tree for repository in Architecture.md * PR Changes * Implemented PR comments * Removed unused file * 🎨 Auto-generated directory tree for repository in Architecture.md --------- Co-authored-by: alantao912 <alantao912@users.noreply.github.com>
- Loading branch information
1 parent
f9b7d92
commit 35b6972
Showing
14 changed files
with
1,916 additions
and
5,977 deletions.
There are no files selected for viewing
Submodule dlp-cli
updated
20 files
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
29 changes: 29 additions & 0 deletions
29
serverless/packages/functions/src/datasets/user/delete_url.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
import { APIGatewayProxyHandlerV2 } from "aws-lambda"; | ||
import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; | ||
import { S3Client, DeleteObjectCommand } from "@aws-sdk/client-s3"; | ||
import parseJwt from "@dlp-sst-app/core/parseJwt"; | ||
|
||
export const handler: APIGatewayProxyHandlerV2 = async (event) => { | ||
if (event?.pathParameters?.type && event.pathParameters.filename) { | ||
const client = new S3Client(); | ||
const user_id: string = parseJwt(event.headers.authorization ?? "")[ | ||
"user_id" | ||
]; | ||
// Generate the presigned URL for a putObject operation | ||
const command = new DeleteObjectCommand({ | ||
Bucket: "dlp-upload-bucket", | ||
Key: `${user_id}/${event.pathParameters.type}/${event.pathParameters.filename}`, | ||
}); | ||
const url = await getSignedUrl(client, command, { | ||
expiresIn: 15 * 60, | ||
}); | ||
return { | ||
statusCode: 200, | ||
body: JSON.stringify({ data: url, message: "Success" }), | ||
}; | ||
} | ||
return { | ||
statusCode: 404, | ||
body: JSON.stringify({ message: "Not Found" }), | ||
}; | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
|
||
export enum TrainStatus { | ||
QUEUED = "QUEUED", | ||
STARTING="STARTING", | ||
UPLOADING="UPLOADING", | ||
TRAINING="TRAINING", | ||
SUCCESS="SUCCESS", | ||
ERROR="ERROR" | ||
} |
63 changes: 63 additions & 0 deletions
63
serverless/packages/functions/src/trainspace/create_image_trainspace.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,63 @@ | ||
import { APIGatewayProxyHandlerV2 } from "aws-lambda"; | ||
import parseJwt from "@dlp-sst-app/core/parseJwt"; | ||
import { v4 as uuidv4 } from 'uuid'; | ||
import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; | ||
import { DynamoDBDocumentClient, PutCommand, PutCommandInput } from '@aws-sdk/lib-dynamodb'; | ||
import { TrainStatus } from './constants'; | ||
|
||
export const handler: APIGatewayProxyHandlerV2 = async (event) => { | ||
if (event) { | ||
const user_id: string = parseJwt(event.headers.authorization ?? "")["user_id"]; | ||
const eventBody = JSON.parse(event.body? event.body : ""); | ||
|
||
const trainspaceId = uuidv4(); | ||
const putCommandInput: PutCommandInput = { | ||
TableName: "trainspace", | ||
Item: | ||
{ | ||
trainspace_id: trainspaceId, | ||
uid: user_id, | ||
created: Date.now().toString(), | ||
data_source: 'IMAGE', | ||
dataset_data: JSON.stringify(eventBody['dataset_data']), | ||
name: eventBody['name'], | ||
parameters_data: JSON.stringify(eventBody['parameters_data']), | ||
review_data: eventBody['review_data'], | ||
status: TrainStatus.QUEUED | ||
} | ||
} | ||
|
||
if (putCommandInput == null) | ||
{ | ||
return { | ||
statusCode: 400, | ||
body: JSON.stringify({ message: "Invalid request body" }) | ||
} | ||
} | ||
|
||
const client = new DynamoDBClient({}); | ||
const docClient = DynamoDBDocumentClient.from(client); | ||
|
||
const command = new PutCommand(putCommandInput); | ||
const response = await docClient.send(command); | ||
|
||
if (response.$metadata.httpStatusCode != 200) { | ||
client.destroy(); | ||
|
||
return { | ||
statusCode: 500, | ||
body: JSON.stringify({ message: "Internal server error."}) | ||
}; | ||
} | ||
|
||
client.destroy(); | ||
return { | ||
statusCode: 200, | ||
body: JSON.stringify({ trainspaceId: trainspaceId, message: "Successfully created a new trainspace."}) | ||
}; | ||
} | ||
return { | ||
statusCode: 404, | ||
body: JSON.stringify({ message: "Not Found" }), | ||
}; | ||
}; |
67 changes: 67 additions & 0 deletions
67
serverless/packages/functions/src/trainspace/create_tabular_trainspace.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,67 @@ | ||
import { APIGatewayProxyHandlerV2 } from "aws-lambda"; | ||
import parseJwt from "@dlp-sst-app/core/parseJwt"; | ||
import { v4 as uuidv4 } from 'uuid'; | ||
import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; | ||
import { DynamoDBDocumentClient, PutCommand, PutCommandInput } from '@aws-sdk/lib-dynamodb'; | ||
import { TrainStatus } from './constants'; | ||
|
||
export const handler: APIGatewayProxyHandlerV2 = async (event) => { | ||
if (event) { | ||
const user_id: string = parseJwt(event.headers.authorization ?? "")["user_id"]; | ||
const eventBody = JSON.parse(event.body? event.body : ""); | ||
|
||
const trainspaceId = uuidv4(); | ||
let putCommandInput: PutCommandInput = { | ||
TableName: "trainspace", | ||
Item: | ||
{ | ||
trainspace_id: trainspaceId, | ||
uid: user_id, | ||
created: Date.now().toString(), | ||
data_source: 'TABULAR', | ||
dataset_data: JSON.stringify(eventBody['dataset_data']), | ||
name: eventBody['name'], | ||
parameters_data: { | ||
criterion: eventBody['criterion'], | ||
optimizer_name: eventBody['optimizer_name'], | ||
shuffle: eventBody['shuffle'], | ||
epochs: eventBody['epochs'], | ||
batch_size: eventBody['batch_size'], | ||
user_arch: eventBody['user_arch'] | ||
}, | ||
review_data: "", | ||
status: TrainStatus.QUEUED | ||
} | ||
} | ||
|
||
if (putCommandInput == null) | ||
{ | ||
return { | ||
statusCode: 400, | ||
body: JSON.stringify({ message: "Invalid request body" }) | ||
} | ||
} | ||
|
||
const client = new DynamoDBClient({}); | ||
const docClient = DynamoDBDocumentClient.from(client); | ||
|
||
const command = new PutCommand(putCommandInput); | ||
const response = await docClient.send(command); | ||
|
||
if (response.$metadata.httpStatusCode != 200) { | ||
return { | ||
statusCode: 500, | ||
body: JSON.stringify({ message: "Internal server error."}) | ||
}; | ||
} | ||
|
||
return { | ||
statusCode: 200, | ||
body: JSON.stringify({ trainspaceId: trainspaceId, message: "Successfully created a new trainspace."}) | ||
}; | ||
} | ||
return { | ||
statusCode: 404, | ||
body: JSON.stringify({ message: "Not Found" }), | ||
}; | ||
}; |
46 changes: 46 additions & 0 deletions
46
serverless/packages/functions/src/trainspace/delete_trainspace.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,46 @@ | ||
import { APIGatewayProxyHandlerV2 } from "aws-lambda"; | ||
|
||
import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; | ||
import { DynamoDBDocumentClient, DeleteCommand } from '@aws-sdk/lib-dynamodb'; | ||
|
||
export const handler: APIGatewayProxyHandlerV2 = async (event) => { | ||
let queryParams = null; | ||
if (event && (queryParams = event['pathParameters']) != null) { | ||
const trainspaceId: string | undefined = queryParams['id']; | ||
|
||
if (trainspaceId == undefined) { | ||
return { | ||
statusCode: 400, | ||
body: JSON.stringify({ message : "Malformed request content - trainspace ID missing." }), | ||
}; | ||
} | ||
|
||
const client = new DynamoDBClient({}); | ||
const docClient = DynamoDBDocumentClient.from(client); | ||
|
||
const command = new DeleteCommand({ | ||
TableName : "trainspace", | ||
Key : | ||
{ | ||
trainspace_id: trainspaceId | ||
} | ||
}); | ||
|
||
const response = await docClient.send(command); | ||
if (response.$metadata.httpStatusCode == undefined || response.$metadata.httpStatusCode != 200) | ||
{ | ||
return { | ||
statusCode: 404, | ||
body: JSON.stringify({ message : "Delete operation failed" }) | ||
} | ||
} | ||
return { | ||
statusCode: 200, | ||
body: "Successfully deleted trainspace with id " + trainspaceId | ||
} | ||
} | ||
return { | ||
statusCode: 400, | ||
body: JSON.stringify({ message : "Malformed request content" }), | ||
}; | ||
}; |
50 changes: 50 additions & 0 deletions
50
serverless/packages/functions/src/trainspace/get_all_trainspace.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,50 @@ | ||
import { APIGatewayProxyHandlerV2 } from "aws-lambda"; | ||
import parseJwt from "@dlp-sst-app/core/parseJwt"; | ||
import { DynamoDBClient, QueryCommand } from '@aws-sdk/client-dynamodb'; | ||
|
||
export const handler: APIGatewayProxyHandlerV2 = async (event) => { | ||
if (event) { | ||
const user_id: string = parseJwt(event.headers.authorization ?? "")[ | ||
"user_id" | ||
]; | ||
|
||
const client = new DynamoDBClient({}); | ||
|
||
const fetchedTrainspaceIds: Array<string> = []; | ||
let lastEvaluatedKey = undefined; | ||
|
||
do { | ||
const getCommand: QueryCommand = new QueryCommand({ | ||
TableName: "trainspace", | ||
IndexName: "uid", | ||
KeyConditionExpression: "uid = :uid", | ||
ExpressionAttributeValues: { | ||
":uid" : | ||
{ | ||
S: user_id | ||
} | ||
}, | ||
ExclusiveStartKey: lastEvaluatedKey | ||
}); | ||
|
||
const results = await client.send(getCommand); | ||
lastEvaluatedKey = results.LastEvaluatedKey; | ||
|
||
if (results['Items']) { | ||
const page: Array<string | undefined> = results['Items']?.map(trainspace => trainspace['trainspace_id'].S); | ||
page.forEach(id => { if (id) fetchedTrainspaceIds.push(id); }); | ||
} | ||
|
||
|
||
} while (lastEvaluatedKey !== undefined); | ||
return { | ||
statusCode: 200, | ||
body: JSON.stringify({ trainspace_ids : fetchedTrainspaceIds}) | ||
}; | ||
} | ||
|
||
return { | ||
statusCode: 404, | ||
body: JSON.stringify({ message: "Not Found" }), | ||
}; | ||
}; |
46 changes: 46 additions & 0 deletions
46
serverless/packages/functions/src/trainspace/get_trainspace.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,46 @@ | ||
import { APIGatewayProxyHandlerV2, APIGatewayProxyEventV2 } from "aws-lambda"; | ||
import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; | ||
import { DynamoDBDocumentClient, GetCommand } from '@aws-sdk/lib-dynamodb'; | ||
|
||
export const handler: APIGatewayProxyHandlerV2 = async (event : APIGatewayProxyEventV2) => { | ||
const queryParams = event['pathParameters']; | ||
if (queryParams != null) | ||
{ | ||
const trainspaceId: string | undefined = queryParams['id']; | ||
|
||
if (trainspaceId == undefined) { | ||
return { | ||
statusCode: 400, | ||
body: JSON.stringify({message: "Malformed request content - trainspace ID missing."}) | ||
}; | ||
} | ||
|
||
const client: DynamoDBClient = new DynamoDBClient({}); | ||
const docClient = DynamoDBDocumentClient.from(client); | ||
|
||
const command : GetCommand = new GetCommand({ | ||
TableName : "trainspace", | ||
Key : | ||
{ | ||
trainspace_id : trainspaceId | ||
} | ||
}); | ||
|
||
const response = await docClient.send(command); | ||
if (!response.Item) | ||
{ | ||
return { | ||
statusCode: 404, | ||
body: JSON.stringify({message: "Provided trainspaceId does not exist"}) | ||
} | ||
} | ||
return { | ||
statusCode: 200, | ||
body: JSON.stringify({message: "Successfully retrieved trainspace data", trainspace: response.Item}) | ||
} | ||
} | ||
return { | ||
statusCode: 400, | ||
body: JSON.stringify({message: "Malformed request content"}) | ||
}; | ||
}; |
Oops, something went wrong.