chore(dev): reorganize repository (#5153)

.eslintrc.js

@@ -8,9 +8,8 @@ module.exports = {
   parser: '@typescript-eslint/parser',
   parserOptions: {
     project: [
-      './src/tsconfig.json',
-      './src/bp/ui-*/tsconfig.json',
-      './src/bp/admin/ui/tsconfig.json',
+      './packages/*/tsconfig.json',
+      './packages/bp/src/admin/ui/tsconfig.json',
       './modules/tsconfig*.eslint.json',
       './internal-modules/tsconfig*.eslint.json',
       './build/module-builder/tsconfig.json',

.github/workflows/code/test-migrations/index.js

@@ -1,6 +1,6 @@
 require('bluebird-global')
 const exec = require('child_process').exec
-const archive = require('../../../../out/bp/core/misc/archive')
+const archive = require('../../../../packages/bp/dist/core/misc/archive')
 const fs = require('fs')
 const rimraf = require('rimraf')
 const path = require('path')
@@ -64,8 +64,8 @@ const start = async () => {
 }
 
 const prepareDataFolder = async buffer => {
-  await Promise.fromCallback(cb => rimraf('./out/bp/data', cb))
-  await archive.extractArchive(buffer, './out/bp/data')
+  await Promise.fromCallback(cb => rimraf('./packages/bp/dist/data', cb))
+  await archive.extractArchive(buffer, './packages/bp/dist/data')
 
   if (isPostgresDb()) {
     await restorePostgresDump()
@@ -74,7 +74,7 @@ const prepareDataFolder = async buffer => {
 
 const restorePostgresDump = async () => {
   const dbUrl = process.env.DATABASE_URL
-  const dumpPath = path.resolve('./out/bp/data/storage/postgres.dump')
+  const dumpPath = path.resolve('./packages/bp/dist/data/storage/postgres.dump')
 
   console.log('Restoring Postgres dump file...')
 
@@ -108,11 +108,11 @@ const testMigration = async (botName, startVersion, targetVersion, { isDown }) =
 }
 
 const isPostgresDb = () => {
-  return fs.existsSync(path.resolve('./out/bp/data/storage/postgres.dump'))
+  return fs.existsSync(path.resolve('./packages/bp/dist/data/storage/postgres.dump'))
 }
 
 const getMostRecentVersion = () => {
-  const coreMigrations = getMigrations('./out/bp')
+  const coreMigrations = getMigrations('./packages/bp/dist')
   const modules = fs.readdirSync('./modules')
 
   const moduleMigrations = _.flatMap(modules, module => getMigrations(`./modules/${module}/dist`))

.github/workflows/docker.yml

@@ -22,13 +22,13 @@ jobs:
           yarn --force --ignore-engines
           yarn run build --linux --prod --verbose
           yarn run package --linux
-          cp build/docker/Dockerfile out/binaries/
+          cp build/docker/Dockerfile packages/bp/binaries/
       - name: Docker build and push
         uses: docker/build-push-action@v1
         with:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
           registry: docker.pkg.github.com
           repository: botpress/botpress/botpress
-          path: out/binaries
+          path: packages/bp/binaries
           tag_with_sha: true

.github/workflows/migration-tests.yml

@@ -33,7 +33,7 @@ jobs:
           node-version: '12.13.0'
       - name: Fetch Node Packages
         run: |
-          yarn add @actions/core @actions/github aws-sdk
+          yarn add -W @actions/core @actions/github aws-sdk
       - name: Build core, UI and modules
         run: |
           NODE_OPTIONS="--max-old-space-size=6000" yarn build

.gitignore

@@ -45,6 +45,13 @@ test-report.html
 metadata.json
 edition.json
 /pro
+packages/bp/binaries/
+packages/bp/archives
+
+# Reorg
+packages/bp/src/pro
+packages/bp/src/ml
+src/bp
 
 build/tests/e2e/screenshots/*
 !build/tests/e2e/screenshots/.gitkeep

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "src/bp/pro"]
-	path = src/bp/pro
+[submodule "pro"]
+	path = packages/bp/src/pro
 	url = git@github.com:botpress/botpress-pro.git

.vscode/launch.json

@@ -6,12 +6,12 @@
       "runtimeVersion": "12.13.0",
       "request": "launch",
       "name": "Debug App",
-      "program": "${workspaceFolder}/out/bp/index.js",
-      "cwd": "${workspaceFolder}/out/bp",
+      "program": "${workspaceFolder}/packages/bp/dist/index.js",
+      "cwd": "${workspaceFolder}/packages/bp/dist",
       "args": ["start"],
       "env": {
         "BP_MODULES_PATH": "${workspaceFolder}/modules:${workspaceFolder}/internal-modules",
-        "NODE_PATH": "${workspaceFolder}/out/bp",
+        "NODE_PATH": "${workspaceFolder}/packages/bp/dist",
         "DEBUG": "bp:none",
         "BLUEBIRD_DEBUG": "1",
         "TELEMETRY_URL": "https://telemetry.botpress.dev/ingest"
@@ -33,7 +33,7 @@
       "runtimeVersion": "12.13.0",
       "request": "launch",
       "name": "Debug Stan",
-      "program": "${workspaceFolder}/out/bp/index.js",
+      "program": "${workspaceFolder}/packages/bp/dist/index.js",
       "cwd": "${workspaceFolder}",
       "args": [
         "nlu",
@@ -45,7 +45,7 @@
       "protocol": "auto",
       "env": {
         "BP_MODULES_PATH": "${workspaceFolder}/modules:${workspaceFolder}/internal-modules",
-        "NODE_PATH": "${workspaceFolder}/out/bp",
+        "NODE_PATH": "${workspaceFolder}/packages/bp/dist",
         "DEBUG": "bp:none",
         "BLUEBIRD_DEBUG": "1"
       },
@@ -139,7 +139,7 @@
         "BP_MODULES_PATH": "../../modules",
         "NODE_PATH": "./"
       },
-      "cwd": "${workspaceFolder}/out/bp"
+      "cwd": "${workspaceFolder}/packages/bp/dist"
     },
     {
       "name": "(lldb) CRFSuite ",

.vscode/settings.json

@@ -63,11 +63,11 @@
     "string": "cpp",
     "__split_buffer": "cpp"
   },
-  "favorites.resources": ["out/bp/data"],
+  "favorites.resources": ["packages/bp/dist/data"],
   "sqltools.connections": [
     {
       "askForPassword": false,
-      "database": "./out/bp/data/storage/core.sqlite",
+      "database": "./packages/bp/dist/data/storage/core.sqlite",
       "dialect": "SQLite",
       "name": "BP Core DB"
     }

build/downloader/src/index.ts

@@ -31,7 +31,7 @@ yargs
     output: {
       alias: 'o',
       description: 'Choose a different output location',
-      default: path.resolve(__dirname, '../../../out/bp')
+      default: path.resolve(__dirname, '../../../packages/bp/dist')
     }
   })
   .command(['list', '$0'], 'List available and installed versions of a tool', {}, async (argv: any) => {

build/gulp.core.js

@@ -9,7 +9,9 @@ const buildJsonSchemas = require('./jsonschemas')
 const fs = require('fs')
 const mkdirp = require('mkdirp')
 const { exec, spawn } = require('child_process')
-const rimraf = require('gulp-rimraf')
+const gulpRimraf = require('gulp-rimraf')
+const rimraf = require('rimraf')
+require('bluebird-global')
 
 const maybeFetchPro = () => {
   const isProBuild = process.env.EDITION === 'pro' || fs.existsSync('pro')
@@ -31,10 +33,10 @@ const writeMetadata = () => {
 }
 
 const clearMigrations = () => {
-  return gulp.src('./out/bp/migrations/*.*', { allowEmpty: true }).pipe(rimraf())
+  return gulp.src('./packages/bp/dist/migrations/*.*', { allowEmpty: true }).pipe(gulpRimraf())
 }
 
-const tsProject = ts.createProject(path.resolve(__dirname, '../src/tsconfig.json'))
+const tsProject = ts.createProject(path.resolve(__dirname, '../packages/bp/tsconfig.json'))
 const compileTypescript = () => {
   return tsProject
     .src()
@@ -48,18 +50,18 @@ const compileTypescript = () => {
         }
       })
     )
-    .pipe(gulp.dest('./out/bp'))
+    .pipe(gulp.dest('./packages/bp/dist'))
 }
 
 const watch = () => {
-  return gulp.watch(['./src/**/*.ts'], { ignored: ['./src/bp/ui-**'] }, compileTypescript)
+  return gulp.watch(['./packages/bp/src/**/*.ts'], { ignored: ['./src/bp/ui-**'] }, compileTypescript)
 }
 
 const createOutputDirs = () => {
   return gulp
     .src('*.*', { read: false })
-    .pipe(gulp.dest('./out/bp/data'))
-    .pipe(gulp.dest('./out/bp/data/storage'))
+    .pipe(gulp.dest('./packages/bp/dist/data'))
+    .pipe(gulp.dest('./packages/bp/dist/data/storage'))
 }
 
 const buildSchemas = cb => {
@@ -68,15 +70,19 @@ const buildSchemas = cb => {
 }
 
 const copyBinaries = () => {
-  return gulp.src('src/bp/ml/bin/*.*').pipe(gulp.dest('./out/bp/ml/bin'))
+  return gulp.src('src/bp/ml/bin/*.*').pipe(gulp.dest('./packages/bp/dist/ml/bin'))
 }
 
 const copyPreTrained = () => {
-  return gulp.src('src/bp/nlu/engine/assets/pre-trained/*').pipe(gulp.dest('./out/bp/nlu/engine/assets/pre-trained'))
+  return gulp
+    .src('src/bp/nlu/engine/assets/pre-trained/*')
+    .pipe(gulp.dest('./packages/bp/dist/nlu/engine/assets/pre-trained'))
 }
 
 const copyStopWords = () => {
-  return gulp.src('src/bp/nlu/engine/assets/stop-words/*').pipe(gulp.dest('./out/bp/nlu/engine/assets/stop-words'))
+  return gulp
+    .src('src/bp/nlu/engine/assets/stop-words/*')
+    .pipe(gulp.dest('./packages/bp/dist/nlu/engine/assets/stop-words'))
 }
 
 const checkTranslations = cb => {
@@ -114,10 +120,19 @@ const build = () => {
   ])
 }
 
+const cleanup = async () => {
+  await Promise.fromCallback(cb => rimraf('packages/bp/archives', cb))
+  await Promise.fromCallback(cb => rimraf('packages/bp/binaries', cb))
+  await Promise.fromCallback(cb => rimraf('**/dist/**', cb))
+  await Promise.fromCallback(cb => rimraf('out', cb))
+  await Promise.fromCallback(cb => rimraf('**/node_modules/**', cb))
+}
+
 module.exports = {
   build,
   watch,
   checkTranslations,
   buildDownloader,
-  initDownloader
+  initDownloader,
+  cleanup
 }

build/gulp.docs.js

@@ -7,7 +7,7 @@ const cheerio = require('cheerio')
 const fs = require('fs')
 
 const buildRef = () => {
-  return gulp.src(['./src/bp/sdk/botpress.d.ts']).pipe(
+  return gulp.src(['./packages/bp/src/sdk/botpress.d.ts']).pipe(
     typedoc({
       out: './docs/reference/public',
       mode: 'file',
@@ -42,10 +42,10 @@ const alterReference = async () => {
   fs.writeFileSync(path.join(__dirname, '../docs/reference/public/modules/_botpress_sdk_.html'), newFile)
 
   const hrefsToReplace = ['../enums', '../classes', '../interfaces']
-  $('a').map(function () {
+  $('a').map(function() {
     const href = $(this).attr('href')
     if (!href) {
-      return;
+      return
     }
 
     if (href.startsWith('_botpress_sdk')) {
@@ -57,7 +57,9 @@ const alterReference = async () => {
     }
   })
 
-  const fixedContentPaths = $.html().replace('../assets/', 'assets/').replace(/\.\.\/globals.html/g, 'globals.html')
+  const fixedContentPaths = $.html()
+    .replace('../assets/', 'assets/')
+    .replace(/\.\.\/globals.html/g, 'globals.html')
   fs.writeFileSync(path.join(__dirname, '../docs/reference/public/index.html'), fixedContentPaths)
 }
 

build/gulp.migration.js

@@ -17,7 +17,7 @@ const dumpServerData = async () => {
 
   const tmpDir = tmp.dirSync({ unsafeCleanup: true })
 
-  const dataFolder = await getValidPath(['out/bp/data', 'data'], rootPath)
+  const dataFolder = await getValidPath(['packages/bp/dist/data', 'data'], rootPath)
   const dataItems = await fse.readdir(dataFolder)
 
   for (const item of dataItems.filter(x => x !== 'assets')) {
@@ -39,7 +39,7 @@ Please make sure that the credentials provided in DATABASE_URL are valid`)
   const files = await Promise.fromCallback(cb => glob('**/*', { cwd: tmpDir.name, nodir: true, dot: true }, cb))
 
   // Not requiring the file at the top of the file, because it may not have been built yet
-  const { createArchive } = require(path.resolve('./out/bp/core/misc/archive'))
+  const { createArchive } = require(path.resolve('./packages/bp/dist/core/misc/archive'))
 
   const currentVersion = await getVersion(rootPath)
   const filename = await createArchive(`./${archiveName}_${currentVersion}.tgz`, tmpDir.name, files)
@@ -51,7 +51,7 @@ Please make sure that the credentials provided in DATABASE_URL are valid`)
 const dumpPostgresData = async (storagePath, rootPath, pgBinPath) => {
   let databaseUrl = process.env.DATABASE_URL
 
-  const dotEnvPath = await getValidPath(['out/bp/.env', '.env'], rootPath)
+  const dotEnvPath = await getValidPath(['packages/bp/dist/.env', '.env'], rootPath)
   if (dotEnvPath) {
     const dot = dotenv.config({ path: dotEnvPath })
 
@@ -111,12 +111,12 @@ const createMigration = cb => {
 
   const template =
     target === 'core'
-      ? path.resolve(__dirname, '../src/bp/core/migration/templates/template_core.ts')
-      : path.resolve(__dirname, '../src/bp/core/migration/templates/template_module.ts')
+      ? path.resolve(__dirname, '../packages/bp/src/core/migration/templates/template_core.ts')
+      : path.resolve(__dirname, '../packages/bp/src/core/migration/templates/template_module.ts')
 
   const targetDir =
     target === 'core'
-      ? path.resolve(__dirname, '../src/bp/migrations')
+      ? path.resolve(__dirname, '../packages/bp/src/migrations')
       : path.resolve(__dirname, `../modules/${target}/src/migrations`)
 
   const destination = path.resolve(targetDir, `v${version}-${Math.round(Date.now() / 1000)}-${title}.ts`)

build/gulp.modules.js

@@ -68,7 +68,7 @@ Output: ${stdout}`
 
 const packageModule = (modulePath, cb) => {
   exec(
-    `node ../../build/module-builder/bin/entry package -v --out ../../out/binaries/modules/%name%.tgz`,
+    `node ../../build/module-builder/bin/entry package -v --out ../../packages/bp/binaries/modules/%name%.tgz`,
     { cwd: modulePath },
     (err, stdout, stderr) => {
       if (err) {
@@ -126,7 +126,7 @@ const buildModules = () => {
 }
 
 const packageModules = () => {
-  mkdirp.sync('out/binaries/modules')
+  mkdirp.sync('packages/bp/binaries/modules')
   const allModules = getAllModulesRoot()
 
   const command = process.argv[process.argv.length - 2]
@@ -166,15 +166,15 @@ const build = () => {
 
 const cleanModuleAssets = () => {
   const moduleName = _.last(process.argv)
-  return gulp.src(`./out/bp/data/assets/modules/${moduleName}`, { allowEmpty: true }).pipe(rimraf())
+  return gulp.src(`./packages/bp/dist/data/assets/modules/${moduleName}`, { allowEmpty: true }).pipe(rimraf())
 }
 
 const createModuleSymlink = () => {
   const moduleFolder = process.argv.includes('--internal') ? 'internal-modules' : 'modules'
   const moduleName = _.last(process.argv)
   return gulp
     .src(`./${moduleFolder}/${moduleName}/assets/`)
-    .pipe(symlink(`./out/bp/data/assets/modules/${moduleName}/`, { type: 'dir' }))
+    .pipe(symlink(`./packages/bp/dist/data/assets/modules/${moduleName}/`, { type: 'dir' }))
 }
 
 const createAllModulesSymlink = () => {
@@ -188,11 +188,11 @@ const createAllModulesSymlink = () => {
     gulp.task(
       taskName,
       gulp.series(
-        () => gulp.src(`./out/bp/data/assets/modules/${moduleName}`, { allowEmpty: true }).pipe(rimraf()),
+        () => gulp.src(`./packages/bp/dist/data/assets/modules/${moduleName}`, { allowEmpty: true }).pipe(rimraf()),
         () =>
           gulp
             .src(`./${moduleFolder}/${moduleName}/assets/`)
-            .pipe(symlink(`./out/bp/data/assets/modules/${moduleName}/`, { type: 'dir' }))
+            .pipe(symlink(`./packages/bp/dist/data/assets/modules/${moduleName}/`, { type: 'dir' }))
       )
     )
 
@@ -228,10 +228,10 @@ Example: 'yarn watch:modules --m channel-web,nlu,qna' or 'yarn watch:modules --a
 
   modules.forEach(moduleName => {
     try {
-      gulp.src(`./out/bp/data/assets/modules/${moduleName}`, { allowEmpty: true }).pipe(rimraf())
+      gulp.src(`./packages/bp/dist/data/assets/modules/${moduleName}`, { allowEmpty: true }).pipe(rimraf())
       gulp
         .src(`./modules/${moduleName}/assets/`)
-        .pipe(symlink(`./out/bp/data/assets/modules/${moduleName}/`, { type: 'dir' }))
+        .pipe(symlink(`./packages/bp/dist/data/assets/modules/${moduleName}/`, { type: 'dir' }))
     } catch (err) {
       console.log('Cant create symlink for', moduleName)
     }