From f42812eb2553478738c6ba7d380f9fd41b2c22b9 Mon Sep 17 00:00:00 2001 From: Aidan Jones Date: Wed, 16 Oct 2024 10:52:16 -0400 Subject: [PATCH 01/28] Add documentation comment to Session.user.roles --- source/SIL.AppBuilder.Portal/src/auth.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/source/SIL.AppBuilder.Portal/src/auth.ts b/source/SIL.AppBuilder.Portal/src/auth.ts index 6b16957f6..264f9efc5 100644 --- a/source/SIL.AppBuilder.Portal/src/auth.ts +++ b/source/SIL.AppBuilder.Portal/src/auth.ts @@ -10,6 +10,7 @@ declare module '@auth/sveltekit' { interface Session { user: { userId: number; + /** [organizationId, RoleId][]*/ roles: [number, number][]; } & DefaultSession['user']; } From ad35026d727204fc345785716b137ef21bef2fcc Mon Sep 17 00:00:00 2001 From: Aidan Jones Date: Mon, 30 Sep 2024 11:55:27 -0400 Subject: [PATCH 02/28] Methods to interact with BuildEngine --- .../common/build-engine-api/index.ts | 2 + .../common/build-engine-api/requests.ts | 157 ++++++++++++++++++ .../common/build-engine-api/types.ts | 105 ++++++++++++ source/SIL.AppBuilder.Portal/common/index.ts | 1 + 4 files changed, 265 insertions(+) create mode 100644 source/SIL.AppBuilder.Portal/common/build-engine-api/index.ts create mode 100644 source/SIL.AppBuilder.Portal/common/build-engine-api/requests.ts create mode 100644 source/SIL.AppBuilder.Portal/common/build-engine-api/types.ts diff --git a/source/SIL.AppBuilder.Portal/common/build-engine-api/index.ts b/source/SIL.AppBuilder.Portal/common/build-engine-api/index.ts new file mode 100644 index 000000000..4bc332396 --- /dev/null +++ b/source/SIL.AppBuilder.Portal/common/build-engine-api/index.ts @@ -0,0 +1,2 @@ +export * as Types from './types.js'; +export * as Requests from './requests.js'; diff --git a/source/SIL.AppBuilder.Portal/common/build-engine-api/requests.ts b/source/SIL.AppBuilder.Portal/common/build-engine-api/requests.ts new file mode 100644 index 000000000..a21636d39 --- /dev/null +++ b/source/SIL.AppBuilder.Portal/common/build-engine-api/requests.ts @@ -0,0 +1,157 @@ +import prisma from '../prisma.js'; +import * as Types from './types.js'; + +export async function request( + resource: string, + organizationId: number, + method: string = 'GET', + body?: any +) { + const { url, token } = await getURLandToken(organizationId); + return await fetch(`${url}/${resource}`, { + method: method, + headers: { + Authorization: `Bearer ${token}`, + Accept: 'application/json' + }, + body: body ? JSON.stringify(body) : undefined + }); +} +export async function getURLandToken(organizationId: number) { + const org = await prisma.organizations.findUnique({ + where: { + Id: organizationId + }, + select: { + BuildEngineUrl: true, + BuildEngineApiAccessToken: true + } + }); + + return { url: org.BuildEngineUrl, token: org.BuildEngineApiAccessToken }; +} + +export async function systemCheck(organizationId: number) { + return (await request('system/check', organizationId)).status; +} + +export async function createProject( + organizationId: number, + project: Types.ProjectConfig +): Promise { + const res = await request('project', organizationId, 'POST', project); + return res.ok ? ((await res.json()) as Types.ProjectResponse) : ((await res.json()) as Types.ErrorResponse); +} +export async function getProjects( + organizationId: number +): Promise { + const res = await request('project', organizationId); + return res.ok ? ((await res.json()) as Types.ProjectResponse[]) : ((await res.json()) as Types.ErrorResponse); +} +export async function getProject( + organizationId: number, + projectId: number +): Promise { + const res = await request(`project/${projectId}`, organizationId); + return res.ok ? ((await res.json()) as Types.ProjectResponse) : ((await res.json()) as Types.ErrorResponse); +} +export async function deleteProject(organizationId: number, projectId: number): Promise { + const res = await request(`project/${projectId}`, organizationId, 'DELETE'); + return res.status; +} + +export async function getProjectAccessToken( + organizationId: number, + projectId: number, + token: Types.TokenConfig +): Promise { + const res = await request(`project/${projectId}/token`, organizationId, 'POST', token); + return res.ok ? ((await res.json()) as Types.TokenResponse) : ((await res.json()) as Types.ErrorResponse); +} + +export async function createJob( + organizationId: number, + job: Types.JobConfig +): Promise { + const res = await request('job', organizationId, 'POST', job); + return res.ok ? ((await res.json()) as Types.JobResponse) : ((await res.json()) as Types.ErrorResponse); +} +export async function getJobs(organizationId: number): Promise { + const res = await request('job', organizationId); + return res.ok ? ((await res.json()) as Types.JobResponse[]) : ((await res.json()) as Types.ErrorResponse); +} +export async function getJob( + organizationId: number, + jobId: number +): Promise { + const res = await request(`job/${jobId}`, organizationId); + return res.ok ? ((await res.json()) as Types.JobResponse) : ((await res.json()) as Types.ErrorResponse); +} +export async function deleteJob(organizationId: number, jobId: number): Promise { + const res = await request(`job/${jobId}`, organizationId, 'DELETE'); + return res.status; +} + +export async function createBuild( + organizationId: number, + jobId: number, + build: Types.BuildConfig +): Promise { + const res = await request(`job/${jobId}/build`, organizationId, 'POST', build); + return res.ok ? ((await res.json()) as Types.BuildResponse) : ((await res.json()) as Types.ErrorResponse); +} +export async function getBuild( + organizationId: number, + jobId: number, + buildId: number +): Promise { + const res = await request(`job/${jobId}/build/${buildId}`, organizationId); + return res.ok ? ((await res.json()) as Types.BuildResponse) : ((await res.json()) as Types.ErrorResponse); +} +export async function getBuilds( + organizationId: number, + jobId: number +): Promise { + const res = await request(`job/${jobId}/build`, organizationId); + return res.ok ? ((await res.json()) as Types.BuildResponse[]) : ((await res.json()) as Types.ErrorResponse); +} +export async function deleteBuild( + organizationId: number, + jobId: number, + buildId: number +): Promise { + const res = await request(`job/${jobId}/build/${buildId}`, organizationId, 'DELETE'); + return res.status; +} + +export async function createRelease( + organizationId: number, + jobId: number, + buildId: number, + release: Types.ReleaseConfig +): Promise { + const res = await request(`job/${jobId}/build/${buildId}`, organizationId, 'PUT', release); + return res.ok ? ((await res.json()) as Types.ReleaseResponse) : ((await res.json()) as Types.ErrorResponse); +} +export async function getRelease( + organizationId: number, + jobId: number, + buildId: number, + releaseId: number +): Promise { + const res = await request(`job/${jobId}/build/${buildId}/release/${releaseId}`, organizationId); + return res.ok ? ((await res.json()) as Types.ReleaseResponse) : ((await res.json()) as Types.ErrorResponse); +} +export async function deleteRelease( + organizationId: number, + jobId: number, + buildId: number, + releaseId: number +): Promise { + const res = await request( + `job/${jobId}/build/${buildId}/release/${releaseId}`, + organizationId, + 'DELETE' + ); + return res.status; +} diff --git a/source/SIL.AppBuilder.Portal/common/build-engine-api/types.ts b/source/SIL.AppBuilder.Portal/common/build-engine-api/types.ts new file mode 100644 index 000000000..632690af0 --- /dev/null +++ b/source/SIL.AppBuilder.Portal/common/build-engine-api/types.ts @@ -0,0 +1,105 @@ +export type Response = + | ErrorResponse + | ProjectResponse + | TokenResponse + | BuildResponse + | ReleaseResponse; + +export type ErrorResponse = { + responseType: 'error'; + name: string; + message: string; + code: number; + status: number; + type: string; +}; + +type SuccessResponse = { + responseType: 'project' | 'token' | 'job' | 'build' | 'release'; + id: number; + created: Date; + updated: Date; + _links: { + self?: { + href: string; + }; + job?: { + href: string; + }; + }; +}; + +export type ProjectConfig = { + user_id: string; + group_id: string; + app_id: string; + project_name: string; + language_code: string; + publishing_key: string; + storage_type: string; +}; +export type ProjectResponse = SuccessResponse & + ProjectConfig & { + responseType: 'project'; + status: 'initialized' | 'accepted' | 'complete' | 'delete' | 'deleting'; + result: 'SUCCESS' | 'FAILURE' | null; + error: string | null; + url: string; + }; + +export type TokenConfig = { + name: string; +}; +export type TokenResponse = { + responseType: 'token'; + session_token: string; + secret_access_key: string; + access_key_id: string; + expiration: string; + region: string; +}; + +export type JobConfig = { + request_id: string; + git_url: string; + app_id: string; + publisher_id: string; +}; +export type JobResponse = SuccessResponse & + JobConfig & { + responseType: 'job'; + }; + +type BuildCommon = { + targets: string; +}; +export type BuildConfig = BuildCommon & { + environment: { [key: string]: string }; +}; +export type BuildResponse = SuccessResponse & + BuildCommon & { + responseType: 'build'; + job_id: number; + status: 'initialized' | 'accepted' | 'active' | 'expired' | 'postprocessing' | 'completed'; + result: 'SUCCESS' | 'FAILURE' | 'ABORTED' | null; + error: string | null; + artifacts: { [key: string]: string }; + }; + +type ReleaseCommon = { + channel: 'production' | 'beta' | 'alpha'; + targets: string; +}; +export type ReleaseConfig = ReleaseCommon & { + environment: { [key: string]: string }; +}; +export type ReleaseResponse = SuccessResponse & + ReleaseCommon & { + responseType: 'release'; + buildId: number; + status: 'initialized' | 'accepted' | 'active' | 'expired' | 'completed' | 'postprocessing'; + result: 'SUCCESS' | 'FAILURE' | 'EXCEPTION' | null; + error: string | null; + console_text: string; + artifacts: { [key: string]: string }; + }; diff --git a/source/SIL.AppBuilder.Portal/common/index.ts b/source/SIL.AppBuilder.Portal/common/index.ts index 0ad9a043c..2b3b36e5b 100644 --- a/source/SIL.AppBuilder.Portal/common/index.ts +++ b/source/SIL.AppBuilder.Portal/common/index.ts @@ -3,3 +3,4 @@ export { scriptoriaQueue } from './bullmq.js'; export { default as DatabaseWrites } from './databaseProxy/index.js'; export { readonlyPrisma as prisma } from './prisma.js'; export { Workflow } from './workflow/index.js'; +export * as BuildEngine from './build-engine-api/index.js'; From e9d70eac3a61852e96d325417a16f61eaaab3138 Mon Sep 17 00:00:00 2001 From: Aidan Jones Date: Tue, 1 Oct 2024 09:14:44 -0400 Subject: [PATCH 03/28] Get default url and token from env --- .../common/build-engine-api/requests.ts | 65 ++++++++++++++----- 1 file changed, 50 insertions(+), 15 deletions(-) diff --git a/source/SIL.AppBuilder.Portal/common/build-engine-api/requests.ts b/source/SIL.AppBuilder.Portal/common/build-engine-api/requests.ts index a21636d39..6914fb45a 100644 --- a/source/SIL.AppBuilder.Portal/common/build-engine-api/requests.ts +++ b/source/SIL.AppBuilder.Portal/common/build-engine-api/requests.ts @@ -24,11 +24,20 @@ export async function getURLandToken(organizationId: number) { }, select: { BuildEngineUrl: true, - BuildEngineApiAccessToken: true + BuildEngineApiAccessToken: true, + UseDefaultBuildEngine: true } }); - return { url: org.BuildEngineUrl, token: org.BuildEngineApiAccessToken }; + return org.UseDefaultBuildEngine + ? { + url: process.env.DEFAULT_BUILDENGINE_URL, + token: process.env.DEFAULT_BUILDENGINE_API_ACCESS_TOKEN + } + : { + url: org.BuildEngineUrl, + token: org.BuildEngineApiAccessToken + }; } export async function systemCheck(organizationId: number) { @@ -40,20 +49,26 @@ export async function createProject( project: Types.ProjectConfig ): Promise { const res = await request('project', organizationId, 'POST', project); - return res.ok ? ((await res.json()) as Types.ProjectResponse) : ((await res.json()) as Types.ErrorResponse); + return res.ok + ? ((await res.json()) as Types.ProjectResponse) + : ((await res.json()) as Types.ErrorResponse); } export async function getProjects( organizationId: number ): Promise { const res = await request('project', organizationId); - return res.ok ? ((await res.json()) as Types.ProjectResponse[]) : ((await res.json()) as Types.ErrorResponse); + return res.ok + ? ((await res.json()) as Types.ProjectResponse[]) + : ((await res.json()) as Types.ErrorResponse); } export async function getProject( organizationId: number, projectId: number ): Promise { const res = await request(`project/${projectId}`, organizationId); - return res.ok ? ((await res.json()) as Types.ProjectResponse) : ((await res.json()) as Types.ErrorResponse); + return res.ok + ? ((await res.json()) as Types.ProjectResponse) + : ((await res.json()) as Types.ErrorResponse); } export async function deleteProject(organizationId: number, projectId: number): Promise { const res = await request(`project/${projectId}`, organizationId, 'DELETE'); @@ -66,7 +81,9 @@ export async function getProjectAccessToken( token: Types.TokenConfig ): Promise { const res = await request(`project/${projectId}/token`, organizationId, 'POST', token); - return res.ok ? ((await res.json()) as Types.TokenResponse) : ((await res.json()) as Types.ErrorResponse); + return res.ok + ? ((await res.json()) as Types.TokenResponse) + : ((await res.json()) as Types.ErrorResponse); } export async function createJob( @@ -74,18 +91,26 @@ export async function createJob( job: Types.JobConfig ): Promise { const res = await request('job', organizationId, 'POST', job); - return res.ok ? ((await res.json()) as Types.JobResponse) : ((await res.json()) as Types.ErrorResponse); + return res.ok + ? ((await res.json()) as Types.JobResponse) + : ((await res.json()) as Types.ErrorResponse); } -export async function getJobs(organizationId: number): Promise { +export async function getJobs( + organizationId: number +): Promise { const res = await request('job', organizationId); - return res.ok ? ((await res.json()) as Types.JobResponse[]) : ((await res.json()) as Types.ErrorResponse); + return res.ok + ? ((await res.json()) as Types.JobResponse[]) + : ((await res.json()) as Types.ErrorResponse); } export async function getJob( organizationId: number, jobId: number ): Promise { const res = await request(`job/${jobId}`, organizationId); - return res.ok ? ((await res.json()) as Types.JobResponse) : ((await res.json()) as Types.ErrorResponse); + return res.ok + ? ((await res.json()) as Types.JobResponse) + : ((await res.json()) as Types.ErrorResponse); } export async function deleteJob(organizationId: number, jobId: number): Promise { const res = await request(`job/${jobId}`, organizationId, 'DELETE'); @@ -98,7 +123,9 @@ export async function createBuild( build: Types.BuildConfig ): Promise { const res = await request(`job/${jobId}/build`, organizationId, 'POST', build); - return res.ok ? ((await res.json()) as Types.BuildResponse) : ((await res.json()) as Types.ErrorResponse); + return res.ok + ? ((await res.json()) as Types.BuildResponse) + : ((await res.json()) as Types.ErrorResponse); } export async function getBuild( organizationId: number, @@ -106,14 +133,18 @@ export async function getBuild( buildId: number ): Promise { const res = await request(`job/${jobId}/build/${buildId}`, organizationId); - return res.ok ? ((await res.json()) as Types.BuildResponse) : ((await res.json()) as Types.ErrorResponse); + return res.ok + ? ((await res.json()) as Types.BuildResponse) + : ((await res.json()) as Types.ErrorResponse); } export async function getBuilds( organizationId: number, jobId: number ): Promise { const res = await request(`job/${jobId}/build`, organizationId); - return res.ok ? ((await res.json()) as Types.BuildResponse[]) : ((await res.json()) as Types.ErrorResponse); + return res.ok + ? ((await res.json()) as Types.BuildResponse[]) + : ((await res.json()) as Types.ErrorResponse); } export async function deleteBuild( organizationId: number, @@ -131,7 +162,9 @@ export async function createRelease( release: Types.ReleaseConfig ): Promise { const res = await request(`job/${jobId}/build/${buildId}`, organizationId, 'PUT', release); - return res.ok ? ((await res.json()) as Types.ReleaseResponse) : ((await res.json()) as Types.ErrorResponse); + return res.ok + ? ((await res.json()) as Types.ReleaseResponse) + : ((await res.json()) as Types.ErrorResponse); } export async function getRelease( organizationId: number, @@ -140,7 +173,9 @@ export async function getRelease( releaseId: number ): Promise { const res = await request(`job/${jobId}/build/${buildId}/release/${releaseId}`, organizationId); - return res.ok ? ((await res.json()) as Types.ReleaseResponse) : ((await res.json()) as Types.ErrorResponse); + return res.ok + ? ((await res.json()) as Types.ReleaseResponse) + : ((await res.json()) as Types.ErrorResponse); } export async function deleteRelease( organizationId: number, From fb15bbeb2793c37b8d2aa6e3cb7ac97ae778ed43 Mon Sep 17 00:00:00 2001 From: Aidan Jones Date: Tue, 1 Oct 2024 11:36:47 -0400 Subject: [PATCH 04/28] Update job execution to Micah's suggestion --- .../common/BullJobTypes.ts | 8 +- .../node-server/BullWorker.ts | 109 ++---------------- .../node-server/job-executors/base.ts | 7 ++ .../node-server/job-executors/index.ts | 2 + .../job-executors/reassignUserTasks.ts | 99 ++++++++++++++++ .../node-server/job-executors/test.ts | 13 +++ 6 files changed, 136 insertions(+), 102 deletions(-) create mode 100644 source/SIL.AppBuilder.Portal/node-server/job-executors/base.ts create mode 100644 source/SIL.AppBuilder.Portal/node-server/job-executors/index.ts create mode 100644 source/SIL.AppBuilder.Portal/node-server/job-executors/reassignUserTasks.ts create mode 100644 source/SIL.AppBuilder.Portal/node-server/job-executors/test.ts diff --git a/source/SIL.AppBuilder.Portal/common/BullJobTypes.ts b/source/SIL.AppBuilder.Portal/common/BullJobTypes.ts index 7f6906c57..982e870e2 100644 --- a/source/SIL.AppBuilder.Portal/common/BullJobTypes.ts +++ b/source/SIL.AppBuilder.Portal/common/BullJobTypes.ts @@ -13,4 +13,10 @@ export interface SyncUserTasksJob { projectId: number; } -export type ScriptoriaJob = TestJob | SyncUserTasksJob; +export type ScriptoriaJob = JobTypeMap[keyof JobTypeMap]; + +export type JobTypeMap = { + [ScriptoriaJobType.Test]: TestJob; + [ScriptoriaJobType.ReassignUserTasks]: SyncUserTasksJob; + // Add more mappings here as needed +}; diff --git a/source/SIL.AppBuilder.Portal/node-server/BullWorker.ts b/source/SIL.AppBuilder.Portal/node-server/BullWorker.ts index 074abd29b..15ae6ea3e 100644 --- a/source/SIL.AppBuilder.Portal/node-server/BullWorker.ts +++ b/source/SIL.AppBuilder.Portal/node-server/BullWorker.ts @@ -1,6 +1,6 @@ import { Job, Worker } from 'bullmq'; -import { BullMQ, DatabaseWrites, prisma } from 'sil.appbuilder.portal.common'; -import { RoleId } from 'sil.appbuilder.portal.common/prisma'; +import { BullMQ } from 'sil.appbuilder.portal.common'; +import * as Executor from './job-executors/index.js'; export abstract class BullWorker { public worker: Worker; @@ -20,105 +20,12 @@ export class ScriptoriaWorker extends BullWorker { } async run(job: Job): Promise { switch (job.data.type) { - case BullMQ.ScriptoriaJobType.Test: { - job.updateProgress(50); - const time = job.data.time; - await new Promise((r) => setTimeout(r, 1000 * time)); - job.updateProgress(100); - return 0; - } - case BullMQ.ScriptoriaJobType.ReassignUserTasks: { - // TODO: Noop - // Should - // Clear preexecuteentries (product transition steps) - // Remove relevant user tasks - // Create new user tasks (send notifications) - // Recreate preexecute entries - const products = await prisma.products.findMany({ - where: { - ProjectId: job.data.projectId - }, - include: { - ProductTransitions: true - } - }); - for (const product of products) { - // Clear PreExecuteEntries - await DatabaseWrites.productTransitions.deleteMany({ - where: { - WorkflowUserId: null, - ProductId: product.Id, - DateTransition: null - } - }); - // Clear existing UserTasks - await DatabaseWrites.userTasks.deleteMany({ - where: { - ProductId: product.Id - } - }); - // Create tasks for all users that could perform this activity - // TODO: this comes from dwkit GetAllActorsFor(Direct|Reverse)CommandTransitions - const organizationId = ( - await prisma.projects.findUnique({ - where: { - Id: job.data.projectId - }, - include: { - Organization: true - } - }) - ).OrganizationId; - // All users that own the project or are org admins - const allUsersWithAction = await prisma.users.findMany({ - where: { - OR: [ - { - UserRoles: { - some: { - OrganizationId: organizationId, - RoleId: RoleId.OrgAdmin - } - } - }, - { - Projects: { - some: { - Id: job.data.projectId - } - } - } - ] - } - }); - // TODO: DWKit: Need ActivityName and Status from dwkit implementation - const createdTasks = allUsersWithAction.map((user) => ({ - UserId: user.Id, - ProductId: product.Id, - ActivityName: null, - Status: null - })); - await DatabaseWrites.userTasks.createMany({ - data: createdTasks - }); - for (const task of createdTasks) { - // Send notification for the new task - // TODO - // sendNotification(task); - } - // TODO: DWKit: CreatePreExecuteEntries - } - - return ( - await prisma.userTasks.findMany({ - where: { - Product: { - ProjectId: job.data.projectId - } - } - }) - ).length; - } + case BullMQ.ScriptoriaJobType.Test: + return new Executor.Test().execute(job as Job); + case BullMQ.ScriptoriaJobType.ReassignUserTasks: + return new Executor.ReassignUserTasks().execute( + job as Job + ); } } } diff --git a/source/SIL.AppBuilder.Portal/node-server/job-executors/base.ts b/source/SIL.AppBuilder.Portal/node-server/job-executors/base.ts new file mode 100644 index 000000000..91e40e816 --- /dev/null +++ b/source/SIL.AppBuilder.Portal/node-server/job-executors/base.ts @@ -0,0 +1,7 @@ +import { BullMQ } from 'sil.appbuilder.portal.common'; +import { Job } from 'bullmq'; + +export abstract class ScriptoriaJobExecutor { + constructor() {} + abstract execute(job: Job): Promise; +} diff --git a/source/SIL.AppBuilder.Portal/node-server/job-executors/index.ts b/source/SIL.AppBuilder.Portal/node-server/job-executors/index.ts new file mode 100644 index 000000000..2085f42f8 --- /dev/null +++ b/source/SIL.AppBuilder.Portal/node-server/job-executors/index.ts @@ -0,0 +1,2 @@ +export { Test } from './test.js'; +export { ReassignUserTasks } from './reassignUserTasks.js'; \ No newline at end of file diff --git a/source/SIL.AppBuilder.Portal/node-server/job-executors/reassignUserTasks.ts b/source/SIL.AppBuilder.Portal/node-server/job-executors/reassignUserTasks.ts new file mode 100644 index 000000000..4f45602aa --- /dev/null +++ b/source/SIL.AppBuilder.Portal/node-server/job-executors/reassignUserTasks.ts @@ -0,0 +1,99 @@ +import { BullMQ, prisma, DatabaseWrites } from 'sil.appbuilder.portal.common'; +import { RoleId } from 'sil.appbuilder.portal.common/prisma'; +import { Job } from 'bullmq'; +import { ScriptoriaJobExecutor } from './base.js'; + +export class ReassignUserTasks extends ScriptoriaJobExecutor { + async execute(job: Job): Promise { + // TODO: Noop + // Should + // Clear preexecuteentries (product transition steps) + // Remove relevant user tasks + // Create new user tasks (send notifications) + // Recreate preexecute entries + const products = await prisma.products.findMany({ + where: { + ProjectId: job.data.projectId + }, + include: { + ProductTransitions: true + } + }); + for (const product of products) { + // Clear PreExecuteEntries + await DatabaseWrites.productTransitions.deleteMany({ + where: { + WorkflowUserId: null, + ProductId: product.Id, + DateTransition: null + } + }); + // Clear existing UserTasks + await DatabaseWrites.userTasks.deleteMany({ + where: { + ProductId: product.Id + } + }); + // Create tasks for all users that could perform this activity + // TODO: this comes from dwkit GetAllActorsFor(Direct|Reverse)CommandTransitions + const organizationId = ( + await prisma.projects.findUnique({ + where: { + Id: job.data.projectId + }, + include: { + Organization: true + } + }) + ).OrganizationId; + // All users that own the project or are org admins + const allUsersWithAction = await prisma.users.findMany({ + where: { + OR: [ + { + UserRoles: { + some: { + OrganizationId: organizationId, + RoleId: RoleId.OrgAdmin + } + } + }, + { + Projects: { + some: { + Id: job.data.projectId + } + } + } + ] + } + }); + // TODO: DWKit: Need ActivityName and Status from dwkit implementation + const createdTasks = allUsersWithAction.map((user) => ({ + UserId: user.Id, + ProductId: product.Id, + ActivityName: null, + Status: null + })); + await DatabaseWrites.userTasks.createMany({ + data: createdTasks + }); + for (const task of createdTasks) { + // Send notification for the new task + // TODO + // sendNotification(task); + } + // TODO: DWKit: CreatePreExecuteEntries + } + + return ( + await prisma.userTasks.findMany({ + where: { + Product: { + ProjectId: job.data.projectId + } + } + }) + ).length; + } +} diff --git a/source/SIL.AppBuilder.Portal/node-server/job-executors/test.ts b/source/SIL.AppBuilder.Portal/node-server/job-executors/test.ts new file mode 100644 index 000000000..0769e9d26 --- /dev/null +++ b/source/SIL.AppBuilder.Portal/node-server/job-executors/test.ts @@ -0,0 +1,13 @@ +import { BullMQ } from 'sil.appbuilder.portal.common'; +import { Job } from 'bullmq'; +import { ScriptoriaJobExecutor } from './base.js'; + +export class Test extends ScriptoriaJobExecutor { + async execute(job: Job): Promise { + job.updateProgress(50); + const time = job.data.time; + await new Promise((r) => setTimeout(r, 1000 * time)); + job.updateProgress(100); + return 0; + } +} From feb8d6ecdb4d02597979e5d1fe9b4931abbd9170 Mon Sep 17 00:00:00 2001 From: Aidan Jones Date: Wed, 16 Oct 2024 10:44:49 -0400 Subject: [PATCH 05/28] Spawn jobs from workflow - Create Product - Build Product - Publish Product --- .../common/BullJobTypes.ts | 37 ++++- .../common/build-engine-api/types.ts | 4 +- .../common/workflow/default-workflow.ts | 40 +++-- .../node-server/BullWorker.ts | 16 ++ .../node-server/job-executors/index.ts | 4 +- .../node-server/job-executors/product.ts | 151 ++++++++++++++++++ .../node-server/job-executors/reviewers.ts | 10 ++ 7 files changed, 247 insertions(+), 15 deletions(-) create mode 100644 source/SIL.AppBuilder.Portal/node-server/job-executors/product.ts create mode 100644 source/SIL.AppBuilder.Portal/node-server/job-executors/reviewers.ts diff --git a/source/SIL.AppBuilder.Portal/common/BullJobTypes.ts b/source/SIL.AppBuilder.Portal/common/BullJobTypes.ts index 982e870e2..6101de620 100644 --- a/source/SIL.AppBuilder.Portal/common/BullJobTypes.ts +++ b/source/SIL.AppBuilder.Portal/common/BullJobTypes.ts @@ -1,6 +1,12 @@ +import { Channels } from './build-engine-api/types.js'; + export enum ScriptoriaJobType { Test = 'Test', - ReassignUserTasks = 'ReassignUserTasks' + ReassignUserTasks = 'ReassignUserTasks', + CreateProduct = 'CreateProduct', + BuildProduct = 'BuildProduct', + EmailReviewers = 'EmailReviewers', + PublishProduct = 'PublishProduct' } export interface TestJob { @@ -13,10 +19,39 @@ export interface SyncUserTasksJob { projectId: number; } +export interface CreateProductJob { + type: ScriptoriaJobType.CreateProduct; + productId: string; +} + +export interface BuildProductJob { + type: ScriptoriaJobType.BuildProduct; + productId: string; + targets?: string; + environment: { [key: string]: string }; +} + +export interface EmailReviewersJob { + type: ScriptoriaJobType.EmailReviewers; + productId: string; +} + +export interface PublishProductJob { + type: ScriptoriaJobType.PublishProduct; + productId: string; + channel: Channels; + targets: string; + environment: { [key: string]: any }; +} + export type ScriptoriaJob = JobTypeMap[keyof JobTypeMap]; export type JobTypeMap = { [ScriptoriaJobType.Test]: TestJob; [ScriptoriaJobType.ReassignUserTasks]: SyncUserTasksJob; + [ScriptoriaJobType.CreateProduct]: CreateProductJob; + [ScriptoriaJobType.BuildProduct]: BuildProductJob; + [ScriptoriaJobType.EmailReviewers]: EmailReviewersJob; + [ScriptoriaJobType.PublishProduct]: PublishProductJob; // Add more mappings here as needed }; diff --git a/source/SIL.AppBuilder.Portal/common/build-engine-api/types.ts b/source/SIL.AppBuilder.Portal/common/build-engine-api/types.ts index 632690af0..e8f5f6e9d 100644 --- a/source/SIL.AppBuilder.Portal/common/build-engine-api/types.ts +++ b/source/SIL.AppBuilder.Portal/common/build-engine-api/types.ts @@ -86,8 +86,10 @@ export type BuildResponse = SuccessResponse & artifacts: { [key: string]: string }; }; +export type Channels = 'production' | 'beta' | 'alpha' + type ReleaseCommon = { - channel: 'production' | 'beta' | 'alpha'; + channel: Channels; targets: string; }; export type ReleaseConfig = ReleaseCommon & { diff --git a/source/SIL.AppBuilder.Portal/common/workflow/default-workflow.ts b/source/SIL.AppBuilder.Portal/common/workflow/default-workflow.ts index 988061284..2a5e762a2 100644 --- a/source/SIL.AppBuilder.Portal/common/workflow/default-workflow.ts +++ b/source/SIL.AppBuilder.Portal/common/workflow/default-workflow.ts @@ -11,6 +11,8 @@ import { WorkflowEvent } from '../public/workflow.js'; import { RoleId } from '../public/prisma.js'; +import { scriptoriaQueue } from '../index.js'; +import { ScriptoriaJobType } from '../BullJobTypes.js'; /** * IMPORTANT: READ THIS BEFORE EDITING A STATE MACHINE! @@ -305,9 +307,11 @@ export const DefaultWorkflow = setup({ 'Product Creation': { entry: [ assign({ instructions: 'waiting' }), - () => { - // TODO: hook into build engine - console.log('Creating Product'); + ({ context }) => { + scriptoriaQueue.add(`Create Product #${context.productId}`, { + type: ScriptoriaJobType.CreateProduct, + productId: context.productId + }); } ], on: { @@ -460,9 +464,13 @@ export const DefaultWorkflow = setup({ assign({ instructions: 'waiting' }), - () => { - // TODO: hook into build engine - console.log('Building Product'); + ({ context }) => { + scriptoriaQueue.add(`Build Product #${context.productId}`, { + type: ScriptoriaJobType.BuildProduct, + productId: context.productId, + // TODO: assign targets + environment: context.environment + }); } ], on: { @@ -666,9 +674,11 @@ export const DefaultWorkflow = setup({ user: RoleId.AppBuilder }, guard: { type: 'hasReviewers' }, - actions: () => { - // TODO: connect to backend to email reviewers - console.log('Emailing Reviewers'); + actions: ({ context }) => { + scriptoriaQueue.add(`Email Reviewers (Product: ${context.productId})`, { + type: ScriptoriaJobType.EmailReviewers, + productId: context.productId + }); } } } @@ -676,9 +686,15 @@ export const DefaultWorkflow = setup({ 'Product Publish': { entry: [ assign({ instructions: 'waiting' }), - () => { - // TODO: hook into build engine - console.log('Publishing Product'); + ({ context }) => { + scriptoriaQueue.add(`Publish Product #${context.productId}`, { + type: ScriptoriaJobType.PublishProduct, + productId: context.productId, + // TODO: How should these values be determined? + channel: 'alpha', + targets: 'google-play', + environment: context.environment + }); } ], on: { diff --git a/source/SIL.AppBuilder.Portal/node-server/BullWorker.ts b/source/SIL.AppBuilder.Portal/node-server/BullWorker.ts index 15ae6ea3e..c83cb2ad3 100644 --- a/source/SIL.AppBuilder.Portal/node-server/BullWorker.ts +++ b/source/SIL.AppBuilder.Portal/node-server/BullWorker.ts @@ -26,6 +26,22 @@ export class ScriptoriaWorker extends BullWorker { return new Executor.ReassignUserTasks().execute( job as Job ); + case BullMQ.ScriptoriaJobType.CreateProduct: + return new Executor.CreateProduct().execute( + job as Job + ); + case BullMQ.ScriptoriaJobType.BuildProduct: + return new Executor.BuildProduct().execute( + job as Job + ); + case BullMQ.ScriptoriaJobType.PublishProduct: + return new Executor.PublishProduct().execute( + job as Job + ); + case BullMQ.ScriptoriaJobType.EmailReviewers: + return new Executor.EmailReviewers().execute( + job as Job + ); } } } diff --git a/source/SIL.AppBuilder.Portal/node-server/job-executors/index.ts b/source/SIL.AppBuilder.Portal/node-server/job-executors/index.ts index 2085f42f8..09d6fb2ce 100644 --- a/source/SIL.AppBuilder.Portal/node-server/job-executors/index.ts +++ b/source/SIL.AppBuilder.Portal/node-server/job-executors/index.ts @@ -1,2 +1,4 @@ export { Test } from './test.js'; -export { ReassignUserTasks } from './reassignUserTasks.js'; \ No newline at end of file +export { ReassignUserTasks } from './reassignUserTasks.js'; +export { EmailReviewers } from './reviewers.js'; +export { CreateProduct, BuildProduct, PublishProduct } from './product.js'; diff --git a/source/SIL.AppBuilder.Portal/node-server/job-executors/product.ts b/source/SIL.AppBuilder.Portal/node-server/job-executors/product.ts new file mode 100644 index 000000000..55216c07a --- /dev/null +++ b/source/SIL.AppBuilder.Portal/node-server/job-executors/product.ts @@ -0,0 +1,151 @@ +import { + BullMQ, + prisma, + DatabaseWrites, + BuildEngine, + Workflow +} from 'sil.appbuilder.portal.common'; +import { Job } from 'bullmq'; +import { ScriptoriaJobExecutor } from './base.js'; + +// TODO: What would be a meaningful return? +export class CreateProduct extends ScriptoriaJobExecutor { + async execute(job: Job): Promise { + const productData = await prisma.products.findUnique({ + where: { + Id: job.data.productId + }, + select: { + Project: { + select: { + ApplicationType: { + select: { + Name: true + } + }, + WorkflowProjectUrl: true, + OrganizationId: true + } + }, + Store: { + select: { + Name: true + } + } + } + }); + const response = await BuildEngine.Requests.createJob(productData.Project.OrganizationId, { + request_id: job.data.productId, + git_url: productData.Project.WorkflowProjectUrl, + app_id: productData.Project.ApplicationType.Name, + publisher_id: productData.Store.Name + }); + + if (response.responseType === 'error') { + // TODO: What do I do here? Wait some period of time and retry? + return 0; + } else { + await DatabaseWrites.products.update(job.data.productId, { + WorkflowJobId: response.id, + DateUpdated: new Date().toString() + }); + + const flow = await Workflow.restore(job.data.productId); + + flow.send({ type: 'Product Created', userId: null }); + + return response.id; + } + } +} + +// TODO: What would be a meaningful return? +export class BuildProduct extends ScriptoriaJobExecutor { + async execute(job: Job): Promise { + const productData = await prisma.products.findUnique({ + where: { + Id: job.data.productId + }, + select: { + Project: { + select: { + OrganizationId: true + } + }, + WorkflowJobId: true + } + }); + + const response = await BuildEngine.Requests.createBuild( + productData.Project.OrganizationId, + productData.WorkflowJobId, + { + targets: job.data.targets ?? 'apk play-listing', + environment: job.data.environment + } + ); + + if (response.responseType === 'error') { + const flow = await Workflow.restore(job.data.productId); + // TODO: How best to notify of failure? + flow.send({ type: 'Build Failed', userId: null, comment: response.message }); + return 0; + } else { + await DatabaseWrites.products.update(job.data.productId, { + WorkflowBuildId: response.id, + DateUpdated: new Date().toString() + }); + + // TODO: Recurring job to check build status? + + return response.id; + } + } +} + +// TODO: What would be a meaningful return? +export class PublishProduct extends ScriptoriaJobExecutor { + async execute(job: Job): Promise { + const productData = await prisma.products.findUnique({ + where: { + Id: job.data.productId + }, + select: { + Project: { + select: { + OrganizationId: true + } + }, + WorkflowJobId: true, + WorkflowBuildId: true + } + }); + + const response = await BuildEngine.Requests.createRelease( + productData.Project.OrganizationId, + productData.WorkflowJobId, + productData.WorkflowBuildId, + { + channel: job.data.channel, + targets: job.data.targets, + environment: job.data.environment + } + ); + + if (response.responseType === 'error') { + const flow = await Workflow.restore(job.data.productId); + // TODO: How best to notify of failure? + flow.send({ type: 'Publish Failed', userId: null, comment: response.message }); + return 0; + } else { + await DatabaseWrites.products.update(job.data.productId, { + WorkflowPublishId: response.id, + DateUpdated: new Date().toString() + }); + + // TODO: Recurring job to check publish status? + + return response.id; + } + } +} diff --git a/source/SIL.AppBuilder.Portal/node-server/job-executors/reviewers.ts b/source/SIL.AppBuilder.Portal/node-server/job-executors/reviewers.ts new file mode 100644 index 000000000..32678082f --- /dev/null +++ b/source/SIL.AppBuilder.Portal/node-server/job-executors/reviewers.ts @@ -0,0 +1,10 @@ +import { BullMQ } from 'sil.appbuilder.portal.common'; +import { Job } from 'bullmq'; +import { ScriptoriaJobExecutor } from './base.js'; + +export class EmailReviewers extends ScriptoriaJobExecutor { + async execute(job: Job): Promise { + // TODO: send emails (there is currently no integrated service with which to do so) + return 0; + } +} \ No newline at end of file From ff2c5d6ada959b69a07bf7eaed8179a12c504f85 Mon Sep 17 00:00:00 2001 From: Aidan Jones Date: Thu, 3 Oct 2024 11:12:04 -0400 Subject: [PATCH 06/28] Add jobs to check build and publish status --- .../common/BullJobTypes.ts | 23 +++- .../common/workflow/default-workflow.ts | 23 +++- .../node-server/BullWorker.ts | 8 ++ .../node-server/job-executors/index.ts | 8 +- .../node-server/job-executors/product.ts | 120 ++++++++++++++++-- 5 files changed, 168 insertions(+), 14 deletions(-) diff --git a/source/SIL.AppBuilder.Portal/common/BullJobTypes.ts b/source/SIL.AppBuilder.Portal/common/BullJobTypes.ts index 6101de620..539f91987 100644 --- a/source/SIL.AppBuilder.Portal/common/BullJobTypes.ts +++ b/source/SIL.AppBuilder.Portal/common/BullJobTypes.ts @@ -6,7 +6,9 @@ export enum ScriptoriaJobType { CreateProduct = 'CreateProduct', BuildProduct = 'BuildProduct', EmailReviewers = 'EmailReviewers', - PublishProduct = 'PublishProduct' + PublishProduct = 'PublishProduct', + CheckBuildProduct = 'CheckBuildProduct', + CheckPublishProduct = 'CheckPublishProduct' } export interface TestJob { @@ -44,6 +46,23 @@ export interface PublishProductJob { environment: { [key: string]: any }; } +export interface CheckBuildProductJob { + type: ScriptoriaJobType.CheckBuildProduct; + organizationId: number; + productId: string; + jobId: number; + buildId: number; +} + +export interface CheckPublishProductJob { + type: ScriptoriaJobType.CheckPublishProduct; + organizationId: number; + productId: string; + jobId: number; + buildId: number; + releaseId: number; +} + export type ScriptoriaJob = JobTypeMap[keyof JobTypeMap]; export type JobTypeMap = { @@ -53,5 +72,7 @@ export type JobTypeMap = { [ScriptoriaJobType.BuildProduct]: BuildProductJob; [ScriptoriaJobType.EmailReviewers]: EmailReviewersJob; [ScriptoriaJobType.PublishProduct]: PublishProductJob; + [ScriptoriaJobType.CheckBuildProduct]: CheckBuildProductJob; + [ScriptoriaJobType.CheckPublishProduct]: CheckPublishProductJob; // Add more mappings here as needed }; diff --git a/source/SIL.AppBuilder.Portal/common/workflow/default-workflow.ts b/source/SIL.AppBuilder.Portal/common/workflow/default-workflow.ts index 2a5e762a2..dba36b14f 100644 --- a/source/SIL.AppBuilder.Portal/common/workflow/default-workflow.ts +++ b/source/SIL.AppBuilder.Portal/common/workflow/default-workflow.ts @@ -311,6 +311,13 @@ export const DefaultWorkflow = setup({ scriptoriaQueue.add(`Create Product #${context.productId}`, { type: ScriptoriaJobType.CreateProduct, productId: context.productId + }, + { + attempts: 5, + backoff: { + type: 'exponential', + delay: 5000 // 5 seconds + } }); } ], @@ -470,6 +477,13 @@ export const DefaultWorkflow = setup({ productId: context.productId, // TODO: assign targets environment: context.environment + }, + { + attempts: 5, + backoff: { + type: 'exponential', + delay: 5000 // 5 seconds + } }); } ], @@ -694,6 +708,13 @@ export const DefaultWorkflow = setup({ channel: 'alpha', targets: 'google-play', environment: context.environment + }, + { + attempts: 5, + backoff: { + type: 'exponential', + delay: 5000 // 5 seconds + } }); } ], @@ -782,7 +803,7 @@ export const DefaultWorkflow = setup({ Jump: { actions: [ assign({ - start: ({ context, event }) => event.target + start: ({ event }) => event.target }) ], target: '.Start' diff --git a/source/SIL.AppBuilder.Portal/node-server/BullWorker.ts b/source/SIL.AppBuilder.Portal/node-server/BullWorker.ts index c83cb2ad3..ede5fced3 100644 --- a/source/SIL.AppBuilder.Portal/node-server/BullWorker.ts +++ b/source/SIL.AppBuilder.Portal/node-server/BullWorker.ts @@ -42,6 +42,14 @@ export class ScriptoriaWorker extends BullWorker { return new Executor.EmailReviewers().execute( job as Job ); + case BullMQ.ScriptoriaJobType.CheckBuildProduct: + return new Executor.CheckBuildProduct().execute( + job as Job + ); + case BullMQ.ScriptoriaJobType.CheckPublishProduct: + return new Executor.CheckPublishProduct().execute( + job as Job + ); } } } diff --git a/source/SIL.AppBuilder.Portal/node-server/job-executors/index.ts b/source/SIL.AppBuilder.Portal/node-server/job-executors/index.ts index 09d6fb2ce..f12e2319f 100644 --- a/source/SIL.AppBuilder.Portal/node-server/job-executors/index.ts +++ b/source/SIL.AppBuilder.Portal/node-server/job-executors/index.ts @@ -1,4 +1,10 @@ export { Test } from './test.js'; export { ReassignUserTasks } from './reassignUserTasks.js'; export { EmailReviewers } from './reviewers.js'; -export { CreateProduct, BuildProduct, PublishProduct } from './product.js'; +export { + CreateProduct, + BuildProduct, + PublishProduct, + CheckBuildProduct, + CheckPublishProduct +} from './product.js'; diff --git a/source/SIL.AppBuilder.Portal/node-server/job-executors/product.ts b/source/SIL.AppBuilder.Portal/node-server/job-executors/product.ts index 55216c07a..051244843 100644 --- a/source/SIL.AppBuilder.Portal/node-server/job-executors/product.ts +++ b/source/SIL.AppBuilder.Portal/node-server/job-executors/product.ts @@ -3,7 +3,8 @@ import { prisma, DatabaseWrites, BuildEngine, - Workflow + Workflow, + scriptoriaQueue } from 'sil.appbuilder.portal.common'; import { Job } from 'bullmq'; import { ScriptoriaJobExecutor } from './base.js'; @@ -34,26 +35,28 @@ export class CreateProduct extends ScriptoriaJobExecutor { + async execute(job: Job): Promise { + const response = await BuildEngine.Requests.getBuild( + job.data.organizationId, + job.data.jobId, + job.data.buildId + ); + job.updateProgress(50); + if (response.responseType === 'error') { + throw new Error(response.message); + } else { + // TODO: what does the 'expired' status mean? + if (response.status === 'completed' || response.status === 'expired') { + await scriptoriaQueue.removeRepeatableByKey(job.repeatJobKey); + if (response.error) { + job.log(response.error); + } + const flow = await Workflow.restore(job.data.productId); + if (response.result === 'SUCCESS') { + flow.send({ type: 'Build Successful', userId: null }); + } else { + flow.send({ type: 'Build Failed', userId: null, comment: response.error }); + } + job.updateProgress(100); + return response.id; + } + job.updateProgress(100); + return 0; + } + } +} + // TODO: What would be a meaningful return? export class PublishProduct extends ScriptoriaJobExecutor { async execute(job: Job): Promise { @@ -120,7 +170,7 @@ export class PublishProduct extends ScriptoriaJobExecutor { + async execute(job: Job): Promise { + const response = await BuildEngine.Requests.getRelease( + job.data.organizationId, + job.data.jobId, + job.data.buildId, + job.data.releaseId + ); + job.updateProgress(50); + if (response.responseType === 'error') { + throw new Error(response.message); + } else { + // TODO: what does the 'expired' status mean? + if (response.status === 'completed' || response.status === 'expired') { + await scriptoriaQueue.removeRepeatableByKey(job.repeatJobKey); + if (response.error) { + job.log(response.error); + } + const flow = await Workflow.restore(job.data.productId); + if (response.result === 'SUCCESS') { + flow.send({ type: 'Publish Successful', userId: null }); + } else { + flow.send({ type: 'Publish Failed', userId: null, comment: response.error }); + } + job.updateProgress(100); + return response.id; + } + job.updateProgress(100); + return 0; + } + } +} From 15958c61779864289f191c427aad6481310f42bf Mon Sep 17 00:00:00 2001 From: Aidan Jones Date: Thu, 3 Oct 2024 11:20:31 -0400 Subject: [PATCH 07/28] Add stub for recurring system status check job --- .../common/BullJobTypes.ts | 8 ++++++- .../node-server/BullWorker.ts | 22 ++++++++++++++++++- .../SIL.AppBuilder.Portal/node-server/dev.ts | 6 +++-- .../node-server/index.ts | 6 +++-- .../node-server/job-executors/index.ts | 1 + .../node-server/job-executors/systemStatus.ts | 19 ++++++++++++++++ 6 files changed, 56 insertions(+), 6 deletions(-) create mode 100644 source/SIL.AppBuilder.Portal/node-server/job-executors/systemStatus.ts diff --git a/source/SIL.AppBuilder.Portal/common/BullJobTypes.ts b/source/SIL.AppBuilder.Portal/common/BullJobTypes.ts index 539f91987..c557fc1bb 100644 --- a/source/SIL.AppBuilder.Portal/common/BullJobTypes.ts +++ b/source/SIL.AppBuilder.Portal/common/BullJobTypes.ts @@ -8,7 +8,8 @@ export enum ScriptoriaJobType { EmailReviewers = 'EmailReviewers', PublishProduct = 'PublishProduct', CheckBuildProduct = 'CheckBuildProduct', - CheckPublishProduct = 'CheckPublishProduct' + CheckPublishProduct = 'CheckPublishProduct', + CheckSystemStatuses = 'CheckSystemStatuses' } export interface TestJob { @@ -63,6 +64,10 @@ export interface CheckPublishProductJob { releaseId: number; } +export interface CheckSystemStatusesJob { + type: ScriptoriaJobType.CheckSystemStatuses; +} + export type ScriptoriaJob = JobTypeMap[keyof JobTypeMap]; export type JobTypeMap = { @@ -74,5 +79,6 @@ export type JobTypeMap = { [ScriptoriaJobType.PublishProduct]: PublishProductJob; [ScriptoriaJobType.CheckBuildProduct]: CheckBuildProductJob; [ScriptoriaJobType.CheckPublishProduct]: CheckPublishProductJob; + [ScriptoriaJobType.CheckSystemStatuses]: CheckSystemStatusesJob; // Add more mappings here as needed }; diff --git a/source/SIL.AppBuilder.Portal/node-server/BullWorker.ts b/source/SIL.AppBuilder.Portal/node-server/BullWorker.ts index ede5fced3..0a7fb2431 100644 --- a/source/SIL.AppBuilder.Portal/node-server/BullWorker.ts +++ b/source/SIL.AppBuilder.Portal/node-server/BullWorker.ts @@ -1,5 +1,5 @@ import { Job, Worker } from 'bullmq'; -import { BullMQ } from 'sil.appbuilder.portal.common'; +import { BullMQ, scriptoriaQueue } from 'sil.appbuilder.portal.common'; import * as Executor from './job-executors/index.js'; export abstract class BullWorker { @@ -50,6 +50,26 @@ export class ScriptoriaWorker extends BullWorker { return new Executor.CheckPublishProduct().execute( job as Job ); + case BullMQ.ScriptoriaJobType.CheckSystemStatuses: + return new Executor.CheckSystemStatuses().execute( + job as Job + ); } } } + +export function addDefaultRecurringJobs() { + // Recurring job to check the availability of BuildEngine + scriptoriaQueue.add( + 'Check System Statuses (Recurring)', + { + type: BullMQ.ScriptoriaJobType.CheckSystemStatuses + }, + { + repeat: { + pattern: '*/5 * * * *', // every 5 minutes + key: 'defaultCheckSystemStatuses' + } + } + ); +} diff --git a/source/SIL.AppBuilder.Portal/node-server/dev.ts b/source/SIL.AppBuilder.Portal/node-server/dev.ts index 3c883daff..d383990a6 100644 --- a/source/SIL.AppBuilder.Portal/node-server/dev.ts +++ b/source/SIL.AppBuilder.Portal/node-server/dev.ts @@ -2,13 +2,13 @@ import { createBullBoard } from '@bull-board/api'; import { BullAdapter } from '@bull-board/api/bullAdapter.js'; import { ExpressAdapter } from '@bull-board/express'; import express from 'express'; -import { ScriptoriaWorker } from './BullWorker.js'; +import { ScriptoriaWorker, addDefaultRecurringJobs } from './BullWorker.js'; process.env.NODE_ENV = 'development'; const app = express(); -import { scriptoriaQueue } from 'sil.appbuilder.portal.common'; +import { BullMQ, scriptoriaQueue } from 'sil.appbuilder.portal.common'; const serverAdapter = new ExpressAdapter(); serverAdapter.setBasePath('/'); createBullBoard({ @@ -18,4 +18,6 @@ createBullBoard({ app.use(serverAdapter.getRouter()); app.listen(3000, () => console.log('Dev server started')); +addDefaultRecurringJobs(); + new ScriptoriaWorker(); diff --git a/source/SIL.AppBuilder.Portal/node-server/index.ts b/source/SIL.AppBuilder.Portal/node-server/index.ts index 7e4869b5e..fe8ae05c5 100644 --- a/source/SIL.AppBuilder.Portal/node-server/index.ts +++ b/source/SIL.AppBuilder.Portal/node-server/index.ts @@ -7,7 +7,7 @@ import express, { type NextFunction, type Request, type Response } from 'express import path from 'path'; import { prisma } from 'sil.appbuilder.portal.common'; import { fileURLToPath } from 'url'; -import { ScriptoriaWorker } from './BullWorker.js'; +import { ScriptoriaWorker, addDefaultRecurringJobs } from './BullWorker.js'; // Do not import any functional code from the sveltekit codebase // unless you are positive you know what you are doing @@ -72,7 +72,7 @@ app.get('/healthcheck', (req, res) => { }); // BullMQ variables -import { scriptoriaQueue } from 'sil.appbuilder.portal.common'; +import { BullMQ, scriptoriaQueue } from 'sil.appbuilder.portal.common'; // Running on svelte process right now. Consider putting on new thread // Fine like this if majority of job time is waiting for network requests // If there is much processing it should be moved to another thread @@ -96,3 +96,5 @@ const handler = await import('./build/handler.js'); app.use(handler.handler); app.listen(3000, () => console.log('Server started!')); + +addDefaultRecurringJobs(); diff --git a/source/SIL.AppBuilder.Portal/node-server/job-executors/index.ts b/source/SIL.AppBuilder.Portal/node-server/job-executors/index.ts index f12e2319f..ef03ea87c 100644 --- a/source/SIL.AppBuilder.Portal/node-server/job-executors/index.ts +++ b/source/SIL.AppBuilder.Portal/node-server/job-executors/index.ts @@ -8,3 +8,4 @@ export { CheckBuildProduct, CheckPublishProduct } from './product.js'; +export { CheckSystemStatuses } from './systemStatus.js'; diff --git a/source/SIL.AppBuilder.Portal/node-server/job-executors/systemStatus.ts b/source/SIL.AppBuilder.Portal/node-server/job-executors/systemStatus.ts new file mode 100644 index 000000000..e19c12149 --- /dev/null +++ b/source/SIL.AppBuilder.Portal/node-server/job-executors/systemStatus.ts @@ -0,0 +1,19 @@ +import { BullMQ, prisma, DatabaseWrites } from 'sil.appbuilder.portal.common'; +import { Job } from 'bullmq'; +import { ScriptoriaJobExecutor } from './base.js'; + +export class CheckSystemStatuses extends ScriptoriaJobExecutor { + async execute(job: Job): Promise { + // TODO: Do I use the `SystemStatus` table? Why does this table even exist? It mostly duplicates data from `Organizations` but is completely disconnected from `Organizations`. Can these be consolidated? + const systems = await prisma.systemStatuses.findMany(); + job.updateProgress(10); + //const timestamp = new Date(); + systems.forEach((s, i) => { + // TODO: Not doing anything here until above TODO is resolved + job.updateProgress(10 + (i+1) * 80 / systems.length); + }); + //await prisma.$transaction(systems.map()); + job.updateProgress(100); + return systems.length; + } +} \ No newline at end of file From d0a58626f23f2a914b9e78602d19e3a3eb332a65 Mon Sep 17 00:00:00 2001 From: Aidan Jones Date: Fri, 4 Oct 2024 09:48:18 -0400 Subject: [PATCH 08/28] Add Project Creation --- .../common/BullJobTypes.ts | 18 ++- .../common/build-engine-api/types.ts | 10 +- .../node-server/BullWorker.ts | 8 ++ .../node-server/job-executors/index.ts | 1 + .../node-server/job-executors/project.ts | 101 ++++++++++++++++ .../components/ProjectSelector.svelte | 2 +- .../new/[id=idNumber]/+page.server.ts | 111 ++++++++++++++++++ .../projects/new/[id=idNumber]/+page.svelte | 86 ++++++++++++++ 8 files changed, 330 insertions(+), 7 deletions(-) create mode 100644 source/SIL.AppBuilder.Portal/node-server/job-executors/project.ts create mode 100644 source/SIL.AppBuilder.Portal/src/routes/(authenticated)/projects/new/[id=idNumber]/+page.server.ts create mode 100644 source/SIL.AppBuilder.Portal/src/routes/(authenticated)/projects/new/[id=idNumber]/+page.svelte diff --git a/source/SIL.AppBuilder.Portal/common/BullJobTypes.ts b/source/SIL.AppBuilder.Portal/common/BullJobTypes.ts index c557fc1bb..1b7e9e4c9 100644 --- a/source/SIL.AppBuilder.Portal/common/BullJobTypes.ts +++ b/source/SIL.AppBuilder.Portal/common/BullJobTypes.ts @@ -9,7 +9,9 @@ export enum ScriptoriaJobType { PublishProduct = 'PublishProduct', CheckBuildProduct = 'CheckBuildProduct', CheckPublishProduct = 'CheckPublishProduct', - CheckSystemStatuses = 'CheckSystemStatuses' + CheckSystemStatuses = 'CheckSystemStatuses', + CreateProject = 'CreateProject', + CheckCreateProject = 'CheckCreateProject' } export interface TestJob { @@ -68,6 +70,18 @@ export interface CheckSystemStatusesJob { type: ScriptoriaJobType.CheckSystemStatuses; } +export interface CreateProjectJob { + type: ScriptoriaJobType.CreateProject; + projectId: number; +} + +export interface CheckCreateProjectJob { + type: ScriptoriaJobType.CheckCreateProject; + workflowProjectId: number; + organizationId: number; + projectId: number; +} + export type ScriptoriaJob = JobTypeMap[keyof JobTypeMap]; export type JobTypeMap = { @@ -80,5 +94,7 @@ export type JobTypeMap = { [ScriptoriaJobType.CheckBuildProduct]: CheckBuildProductJob; [ScriptoriaJobType.CheckPublishProduct]: CheckPublishProductJob; [ScriptoriaJobType.CheckSystemStatuses]: CheckSystemStatusesJob; + [ScriptoriaJobType.CreateProject]: CreateProjectJob; + [ScriptoriaJobType.CheckCreateProject]: CheckCreateProjectJob; // Add more mappings here as needed }; diff --git a/source/SIL.AppBuilder.Portal/common/build-engine-api/types.ts b/source/SIL.AppBuilder.Portal/common/build-engine-api/types.ts index e8f5f6e9d..8c822ff95 100644 --- a/source/SIL.AppBuilder.Portal/common/build-engine-api/types.ts +++ b/source/SIL.AppBuilder.Portal/common/build-engine-api/types.ts @@ -30,21 +30,21 @@ type SuccessResponse = { }; export type ProjectConfig = { - user_id: string; - group_id: string; app_id: string; project_name: string; language_code: string; - publishing_key: string; storage_type: string; }; export type ProjectResponse = SuccessResponse & ProjectConfig & { responseType: 'project'; - status: 'initialized' | 'accepted' | 'complete' | 'delete' | 'deleting'; + status: 'initialized' | 'accepted' | 'completed' | 'delete' | 'deleting'; result: 'SUCCESS' | 'FAILURE' | null; error: string | null; url: string; + publishing_key: string; + user_id: string; + group_id: string; }; export type TokenConfig = { @@ -86,7 +86,7 @@ export type BuildResponse = SuccessResponse & artifacts: { [key: string]: string }; }; -export type Channels = 'production' | 'beta' | 'alpha' +export type Channels = 'production' | 'beta' | 'alpha'; type ReleaseCommon = { channel: Channels; diff --git a/source/SIL.AppBuilder.Portal/node-server/BullWorker.ts b/source/SIL.AppBuilder.Portal/node-server/BullWorker.ts index 0a7fb2431..9e1078f78 100644 --- a/source/SIL.AppBuilder.Portal/node-server/BullWorker.ts +++ b/source/SIL.AppBuilder.Portal/node-server/BullWorker.ts @@ -54,6 +54,14 @@ export class ScriptoriaWorker extends BullWorker { return new Executor.CheckSystemStatuses().execute( job as Job ); + case BullMQ.ScriptoriaJobType.CreateProject: + return new Executor.CreateProject().execute( + job as Job + ); + case BullMQ.ScriptoriaJobType.CheckCreateProject: + return new Executor.CheckCreateProject().execute( + job as Job + ); } } } diff --git a/source/SIL.AppBuilder.Portal/node-server/job-executors/index.ts b/source/SIL.AppBuilder.Portal/node-server/job-executors/index.ts index ef03ea87c..f2085768e 100644 --- a/source/SIL.AppBuilder.Portal/node-server/job-executors/index.ts +++ b/source/SIL.AppBuilder.Portal/node-server/job-executors/index.ts @@ -9,3 +9,4 @@ export { CheckPublishProduct } from './product.js'; export { CheckSystemStatuses } from './systemStatus.js'; +export { CreateProject, CheckCreateProject } from './project.js'; diff --git a/source/SIL.AppBuilder.Portal/node-server/job-executors/project.ts b/source/SIL.AppBuilder.Portal/node-server/job-executors/project.ts new file mode 100644 index 000000000..f01f97589 --- /dev/null +++ b/source/SIL.AppBuilder.Portal/node-server/job-executors/project.ts @@ -0,0 +1,101 @@ +import { + BullMQ, + prisma, + DatabaseWrites, + BuildEngine, + Workflow, + scriptoriaQueue +} from 'sil.appbuilder.portal.common'; +import { Job } from 'bullmq'; +import { ScriptoriaJobExecutor } from './base.js'; + +// TODO: What would be a meaningful return? +export class CreateProject extends ScriptoriaJobExecutor { + async execute(job: Job): Promise { + const projectData = await prisma.projects.findUnique({ + where: { + Id: job.data.projectId + }, + select: { + OrganizationId: true, + ApplicationType: { + select: { + Name: true + } + }, + Name: true, + Language: true + } + }); + job.updateProgress(25); + const response = await BuildEngine.Requests.createProject(projectData.OrganizationId, { + app_id: projectData.ApplicationType.Name, + project_name: projectData.Name, + language_code: projectData.Language, + storage_type: 's3' + }); + job.updateProgress(50); + if (response.responseType === 'error') { + job.log(response.message); + throw new Error(`Creation of Project #${job.data.projectId} failed!`); + } else { + await DatabaseWrites.projects.update(job.data.projectId, { + WorkflowProjectId: response.id, + WorkflowAppProjectUrl: `${process.env.UI_URL ?? 'http://localhost:5173'}/projects/${job.data.projectId}`, + DateUpdated: new Date() + }); + job.updateProgress(75); + + await scriptoriaQueue.add( + `Check status of Project #${response.id}`, + { + type: BullMQ.ScriptoriaJobType.CheckCreateProject, + workflowProjectId: response.id, + organizationId: projectData.OrganizationId, + projectId: job.data.projectId + }, + { + repeat: { + pattern: '*/1 * * * *' // every minute + } + } + ); + + job.updateProgress(100); + + return response.id; + } + } +} + +export class CheckCreateProject extends ScriptoriaJobExecutor { + async execute(job: Job): Promise { + const response = await BuildEngine.Requests.getProject( + job.data.organizationId, + job.data.workflowProjectId + ); + job.updateProgress(50); + if (response.responseType === 'error') { + job.log(response.message); + throw new Error(response.message); + } else { + if (response.status === 'completed') { + await scriptoriaQueue.removeRepeatableByKey(job.repeatJobKey); + if (response.error) { + job.log(response.error); + } + else { + await DatabaseWrites.projects.update(job.data.projectId, { + WorkflowProjectUrl: response.url, + DateUpdated: new Date() + }); + } + + job.updateProgress(100); + return response.id; + } + job.updateProgress(100); + return 0; + } + } +} diff --git a/source/SIL.AppBuilder.Portal/src/routes/(authenticated)/projects/components/ProjectSelector.svelte b/source/SIL.AppBuilder.Portal/src/routes/(authenticated)/projects/components/ProjectSelector.svelte index bc773259a..0ff330807 100644 --- a/source/SIL.AppBuilder.Portal/src/routes/(authenticated)/projects/components/ProjectSelector.svelte +++ b/source/SIL.AppBuilder.Portal/src/routes/(authenticated)/projects/components/ProjectSelector.svelte @@ -75,7 +75,7 @@ - diff --git a/source/SIL.AppBuilder.Portal/src/routes/(authenticated)/projects/new/[id=idNumber]/+page.server.ts b/source/SIL.AppBuilder.Portal/src/routes/(authenticated)/projects/new/[id=idNumber]/+page.server.ts new file mode 100644 index 000000000..77f73785a --- /dev/null +++ b/source/SIL.AppBuilder.Portal/src/routes/(authenticated)/projects/new/[id=idNumber]/+page.server.ts @@ -0,0 +1,111 @@ +import { idSchema } from '$lib/valibot'; +import { error } from '@sveltejs/kit'; +import { DatabaseWrites, prisma } from 'sil.appbuilder.portal.common'; +import { fail, superValidate } from 'sveltekit-superforms'; +import { valibot } from 'sveltekit-superforms/adapters'; +import * as v from 'valibot'; +import type { Session } from '@auth/sveltekit'; +import type { Actions, PageServerLoad } from './$types'; +import { RoleId } from 'sil.appbuilder.portal.common/prisma'; +import { scriptoriaQueue, BullMQ } from 'sil.appbuilder.portal.common'; +import { time } from 'console'; + +const projectCreateSchema = v.object({ + name: v.pipe(v.string(), v.minLength(1)), + group: idSchema, + language: v.pipe(v.string(), v.minLength(1)), + type: idSchema, + description: v.nullable(v.string()), + public: v.boolean() +}); + +export const load = (async ({ locals, params }) => { + if (!verifyCanCreateProject((await locals.auth())!, parseInt(params.id))) return error(403); + + const organization = await prisma.organizations.findUnique({ + where: { + Id: parseInt(params.id) + }, + select: { + Groups: { + select: { + Id: true, + Name: true + } + }, + OrganizationProductDefinitions: { + select: { + ProductDefinition: { + select: { + ApplicationTypes: true + } + } + } + }, + PublicByDefault: true + } + }); + + const types = organization?.OrganizationProductDefinitions.map( + (opd) => opd.ProductDefinition.ApplicationTypes + ).reduce((p, c) => { + if (!p.some((e) => e.Id === c.Id)) { + p.push(c); + } + return p; + }, [] as { Id: number; Name: string | null; Description: string | null }[]); + + const form = await superValidate( + { + group: organization?.Groups[0]?.Id ?? undefined, + type: types?.[0].Id ?? undefined, + public: organization?.PublicByDefault ?? undefined + }, + valibot(projectCreateSchema) + ); + return { form, organization, types }; +}) satisfies PageServerLoad; + +export const actions: Actions = { + default: async function (event) { + const session = (await event.locals.auth())!; + if (!verifyCanCreateProject(session, parseInt(event.params.id))) return error(403); + + const form = await superValidate(event.request, valibot(projectCreateSchema)); + // TODO: Return/Display error messages + if (!form.valid) return fail(400, { form, ok: false }); + if (isNaN(parseInt(event.params.id))) return fail(400, { form, ok: false }); + const timestamp = (new Date()).toString(); + const project = await DatabaseWrites.projects.create({ + OrganizationId: parseInt(event.params.id), + Name: form.data.name, + GroupId: form.data.group, + OwnerId: session.user.userId, + Language: form.data.language, + TypeId: form.data.type, + Description: form.data.description ?? '', + DateCreated: timestamp, + DateUpdated: timestamp + // TODO: DateActive? + }); + + if (project !== false) { + scriptoriaQueue.add(`Create Project #${project}`, { + type: BullMQ.ScriptoriaJobType.CreateProject, + projectId: project as number + }); + } + + return { form, ok: project !== false }; + } +}; + +async function verifyCanCreateProject(user: Session, orgId: number) { + // Creating a project is allowed if the user is an OrgAdmin, AppBuilder, or SuperAdmin for the organization + const roles = user.user.roles.filter(([org, role]) => org === orgId).map(([org, role]) => role); + return ( + roles.includes(RoleId.AppBuilder) || + roles.includes(RoleId.OrgAdmin) || + roles.includes(RoleId.SuperAdmin) + ); +} diff --git a/source/SIL.AppBuilder.Portal/src/routes/(authenticated)/projects/new/[id=idNumber]/+page.svelte b/source/SIL.AppBuilder.Portal/src/routes/(authenticated)/projects/new/[id=idNumber]/+page.svelte new file mode 100644 index 000000000..2126eaeeb --- /dev/null +++ b/source/SIL.AppBuilder.Portal/src/routes/(authenticated)/projects/new/[id=idNumber]/+page.svelte @@ -0,0 +1,86 @@ + + +
+
+

{m.project_newProject()}

+
+
+ + +
+
+ + +
+
+ + + +
+
+ + +
+
+