From b24d33c418f0d05a6b2e52798d1a52dea39dbd9c Mon Sep 17 00:00:00 2001 From: MicrowaveDev Date: Thu, 10 Oct 2024 05:02:04 +0100 Subject: [PATCH 1/2] switch to postgres and improve content storage --- app/config.ts | 4 +++- app/modules/communicator/fluence.ts | 2 +- app/modules/content/api.ts | 2 +- app/modules/content/index.ts | 12 ++++++---- app/modules/content/interface.ts | 2 ++ app/modules/database/config.ts | 14 ++++++------ app/modules/database/index.ts | 4 ++++ app/modules/database/interface.ts | 3 +++ app/modules/fileCatalog/index.ts | 29 ++++++++++--------------- app/modules/pin/index.ts | 2 +- app/modules/storage/ipfs-http-client.ts | 1 - docker-compose.yml | 16 ++++++++++++++ test/app.test.ts | 2 +- test/autoActions.test.ts | 2 +- test/databaseValues.test.ts | 2 +- test/fileCatalog.test.ts | 2 +- test/foreignAccounts.test.ts | 2 +- test/group.test.ts | 7 +++--- test/groupCategory.test.ts | 2 +- test/invites.test.ts | 2 +- test/pin.test.ts | 2 +- test/render.test.ts | 2 +- test/staticSiteGenerator.test.ts | 4 ++-- test/storage.test.ts | 2 +- test/telegramClient.test.ts | 2 +- test/twitterClient.test.ts | 2 +- 26 files changed, 75 insertions(+), 51 deletions(-) diff --git a/app/config.ts b/app/config.ts index b1f8a564..fcce7db6 100644 --- a/app/config.ts +++ b/app/config.ts @@ -17,7 +17,9 @@ const modulePacks = { //TODO: refactor modules config module.exports = { databaseModule: 'sql', - databaseConfig: {}, + databaseConfig: { + + }, storageConfig: { implementation: process.env.STORAGE_MODULE || 'js-ipfs', jsNode: { diff --git a/app/modules/communicator/fluence.ts b/app/modules/communicator/fluence.ts index 5eac98c8..702d4520 100644 --- a/app/modules/communicator/fluence.ts +++ b/app/modules/communicator/fluence.ts @@ -5,7 +5,7 @@ module.exports = async (app: IGeesomeApp) => { // let neighbours = await dhtApi.getNeighbours(client, nodeId, 'topic') const fluenceService = new FluenceService(app.ms.accountStorage, {logLevel: null /*'debug'*/}); const peerId = await app.ms.accountStorage.getAccountPeerId('self'); - if(peerId) { + if (peerId) { await fluenceService.initClient({ type: "Ed25519", source: peerId.privKey.bytes diff --git a/app/modules/content/api.ts b/app/modules/content/api.ts index bcbd27a3..f566c267 100644 --- a/app/modules/content/api.ts +++ b/app/modules/content/api.ts @@ -109,7 +109,7 @@ module.exports = (app: IGeesomeApp, contentModule: IGeesomeContentModule) => { }); app.ms.api.onGet('content-by-storage-id/:contentStorageId', async (req, res) => { - res.send(await contentModule.getContentByStorageId(req.params.contentStorageId)); + res.send(await contentModule.getContentByStorageAndUserId(req.params.contentStorageId, req.user.id)); }); app.ms.api.onGet('content-stats/*', async (req, res) => { diff --git a/app/modules/content/index.ts b/app/modules/content/index.ts index 2bffa968..2ec706de 100644 --- a/app/modules/content/index.ts +++ b/app/modules/content/index.ts @@ -57,6 +57,10 @@ function getModule(app: IGeesomeApp) { return app.ms.database.getContentByStorageId(storageId); } + getContentByStorageAndUserId(storageId, userId) { + return app.ms.database.getContentByStorageAndUserId(storageId, userId); + } + getContentByManifestId(storageId) { return app.ms.database.getContentByManifestId(storageId); } @@ -87,7 +91,7 @@ function getModule(app: IGeesomeApp) { let previewData = await this.getPreview({id: content.storageId, size: content.size}, content.extension, content.mimeType); await app.ms.database.updateContent(content.id, previewData); const updatedContent = await this.updateContentManifest({ - ...content['toJSON'](), + ...content.toJSON() as any, ...previewData }); @@ -309,7 +313,7 @@ function getModule(app: IGeesomeApp) { if (propsToUpdate.some(prop => options[prop] && content[prop] !== options[prop])) { await app.ms.database.updateContent(content.id, _.pick(options, propsToUpdate)); await this.updateContentManifest({ - ...content['toJSON'](), + ...content.toJSON() as any, ..._.pick(options, propsToUpdate), }); } @@ -324,7 +328,7 @@ function getModule(app: IGeesomeApp) { } await app.ms.database.updateContent(content.id, updateData); return this.updateContentManifest({ - ...content['toJSON'](), + ...content.toJSON() as any, ...updateData, }); } @@ -718,7 +722,7 @@ function getModule(app: IGeesomeApp) { async createContentByObject(userId, contentObject, options?: { userApiKeyId? }) { const storageId = contentObject.manifestStaticStorageId || contentObject.manifestStorageId; - const dbContent = await app.ms.database.getContentByStorageId(storageId); + const dbContent = await app.ms.database.getContentByStorageAndUserId(storageId, userId); if (dbContent) { return dbContent; } diff --git a/app/modules/content/interface.ts b/app/modules/content/interface.ts index f89a9e2f..52359213 100644 --- a/app/modules/content/interface.ts +++ b/app/modules/content/interface.ts @@ -10,6 +10,8 @@ export default interface IGeesomeContentModule { getContentByStorageId(storageId): Promise; + getContentByStorageAndUserId(storageId, userId): Promise; + getContentByManifestId(storageId): Promise; createContentByObject(userId, contentObject, options?: { groupId?, userApiKeyId? }): Promise; diff --git a/app/modules/database/config.ts b/app/modules/database/config.ts index d290769b..2b35c490 100644 --- a/app/modules/database/config.ts +++ b/app/modules/database/config.ts @@ -9,14 +9,14 @@ const log = require('debug')('geesome:database'); module.exports = { - 'name': 'geesome_node', - 'user': 'geesome', - 'password': 'geesome', + 'name': process.env.DATABASE_PASSWORD || 'geesome_node', + 'user': process.env.DATABASE_USER || 'geesome', + 'password': process.env.DATABASE_DB || 'geesome', 'options': { 'logging': (d) => {/*log(d)*/}, - 'host': 'localhost', - // 'dialect': 'postgres', - 'dialect': 'sqlite', - 'storage': `${process.env.DATA_DIR || 'data'}/core.sqlite` + 'host': process.env.DATABASE_HOST || 'localhost', + 'dialect': 'postgres', + // 'dialect': 'sqlite', + // 'storage': `${process.env.DATA_DIR || 'data'}/core.sqlite` } }; diff --git a/app/modules/database/index.ts b/app/modules/database/index.ts index c1f38956..f846684f 100644 --- a/app/modules/database/index.ts +++ b/app/modules/database/index.ts @@ -335,6 +335,10 @@ class MysqlDatabase implements IGeesomeDatabaseModule { return where; } + getUserContentListByIds(userId, contentIds) { + return this.models.Content.findAll({where: {userId, id: {[Op.in]: contentIds}}}); + } + async getAllContentList(searchString, listParams: IListParams = {}) { this.setDefaultListParamsValues(listParams); const {sortBy, sortDir, limit, offset} = listParams; diff --git a/app/modules/database/interface.ts b/app/modules/database/interface.ts index ff9b10e1..01e36d3e 100644 --- a/app/modules/database/interface.ts +++ b/app/modules/database/interface.ts @@ -38,6 +38,8 @@ export interface IGeesomeDatabaseModule { getContentList(accountAddress, listParams?: IListParams): Promise; + getUserContentListByIds(userId, contentIds): Promise; + getContent(id): Promise; getContentByStorageId(storageId, findByPreviews?): Promise; @@ -169,6 +171,7 @@ export interface IContent { encryptedManifestStorageId?: string; propertiesJson?: string; + toJSON?(): string; } export interface IObject { diff --git a/app/modules/fileCatalog/index.ts b/app/modules/fileCatalog/index.ts index 1dd9be3a..3660bc92 100644 --- a/app/modules/fileCatalog/index.ts +++ b/app/modules/fileCatalog/index.ts @@ -125,7 +125,6 @@ function getModule(app: IGeesomeApp, models) { type = undefined; } console.log('userId', userId, 'parentItemId', parentItemId, 'type', type, 'search', search); - return { list: await this.getFileCatalogItemsList(userId, parentItemId, type, search, listParams), total: await this.getFileCatalogItemsCount(userId, parentItemId, type, search) @@ -337,15 +336,12 @@ function getModule(app: IGeesomeApp, models) { const {limit, offset, sortBy, sortDir} = listParams; const where: any = {userId, type, isDeleted: false}; - if (!_.isUndefined(parentItemId)) { where.parentItemId = parentItemId; } - if (search) { where['name'] = {[Op.like]: search}; } - return models.FileCatalogItem.findAll({ where, order: [[sortBy, sortDir.toUpperCase()]], @@ -355,6 +351,17 @@ function getModule(app: IGeesomeApp, models) { }); } + async getFileCatalogItemsCount(userId, parentItemId, type = null, search = '') { + const where: any = {userId, type, isDeleted: false}; + if (!_.isUndefined(parentItemId)) { + where.parentItemId = parentItemId; + } + if (search) { + where['name'] = {[Op.like]: search}; + } + return models.FileCatalogItem.count({where}); + } + async getFileCatalogItemsByContent(userId, contentId, type = null, listParams: IListParams = {}) { app.ms.database.setDefaultListParamsValues(listParams); const {sortBy, sortDir, limit, offset} = listParams; @@ -367,20 +374,6 @@ function getModule(app: IGeesomeApp, models) { }); } - async getFileCatalogItemsCount(userId, parentItemId, type = null, search = '') { - const where: any = {userId, type, isDeleted: false}; - - if (!_.isUndefined(parentItemId)) { - where.parentItemId = parentItemId; - } - - if (search) { - where['name'] = {[Op.like]: search}; - } - - return models.FileCatalogItem.count({where}); - } - async isFileCatalogItemExistWithContent(userId, parentItemId, contentId) { return models.FileCatalogItem.findOne({where: {userId, parentItemId, contentId}}).then(r => !!r); } diff --git a/app/modules/pin/index.ts b/app/modules/pin/index.ts index 860623de..21a6f19b 100644 --- a/app/modules/pin/index.ts +++ b/app/modules/pin/index.ts @@ -72,7 +72,7 @@ function getModule(app: IGeesomeApp, models) { } async pinByPinata(storageId: string, account: IPinAccount, options?) { - const content = await app.ms.content.getContentByStorageId(storageId); + const content = await app.ms.content.getContentByStorageAndUserId(storageId, account.userId); const hostNodes = await app.ms.storage.remoteNodeAddressList(['tcp']); console.log('hostNodes', hostNodes); return axios diff --git a/app/modules/storage/ipfs-http-client.ts b/app/modules/storage/ipfs-http-client.ts index 404025df..669f74b3 100644 --- a/app/modules/storage/ipfs-http-client.ts +++ b/app/modules/storage/ipfs-http-client.ts @@ -24,7 +24,6 @@ module.exports = async (app: IGeesomeApp) => { //TODO: remove config setting after migration to new ipfs http client await service.node.config.set('Addresses.Swarm', await service.node.config.get('Addresses.Swarm').then(list => list.filter(s => !s.includes('quic')))); await service.node.config.set('Bootstrap', await service.node.config.get('Bootstrap').then(list => list.filter(s => !s.includes('quic')))); - console.log("Addresses.Swarm config", await service.node.config.get('Addresses.Swarm')); if (process.env.IPFS_PROFILE) { await service.node.config.profiles.apply(process.env.IPFS_PROFILE); } diff --git a/docker-compose.yml b/docker-compose.yml index a71c211c..cd34648e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,22 @@ services: - IPFS_PROFILE - DEBUG - SSG_RUNTIME + - DATABASE_PASSWORD=geesome + - DATABASE_USER=geesome + - DATABASE_DB=geesome_node + - DATABASE_HOST=geesome_db:5432 + + postgres: + container_name: geesome_db + image: postgres:14-alpine + ports: + - 5432:5432 + volumes: + - ~/apps/postgres:/var/lib/postgresql/data + environment: + - POSTGRES_PASSWORD=geesome + - POSTGRES_USER=geesome + - POSTGRES_DB=geesome_node client: image: nginx diff --git a/test/app.test.ts b/test/app.test.ts index 4f0f9f4c..1be24e98 100644 --- a/test/app.test.ts +++ b/test/app.test.ts @@ -21,7 +21,7 @@ describe("app", function () { const databaseConfig = { name: 'geesome_test', options: { logging: () => { - }, storage: 'database-test.sqlite' + }, dialect: 'sqlite', storage: 'database-test.sqlite' } }; diff --git a/test/autoActions.test.ts b/test/autoActions.test.ts index b8ccecf0..fd7a14b7 100644 --- a/test/autoActions.test.ts +++ b/test/autoActions.test.ts @@ -23,7 +23,7 @@ describe("autoActions", function () { const databaseConfig = { name: 'geesome_test', options: { logging: () => { - }, storage: 'database-test.sqlite' + }, dialect: 'sqlite', storage: 'database-test.sqlite' } }; diff --git a/test/databaseValues.test.ts b/test/databaseValues.test.ts index 61efe48f..007c0376 100644 --- a/test/databaseValues.test.ts +++ b/test/databaseValues.test.ts @@ -12,7 +12,7 @@ import {IGeesomeDatabaseModule} from "../app/modules/database/interface"; const assert = require('assert'); describe("databaseValues", function () { - const databaseConfig = {name: 'geesome_test', options: {logging: true, storage: 'database-test.sqlite'}}; + const databaseConfig = {name: 'geesome_test', options: {logging: true, dialect: 'sqlite', storage: 'database-test.sqlite'}}; let database: IGeesomeDatabaseModule; diff --git a/test/fileCatalog.test.ts b/test/fileCatalog.test.ts index e8dee413..93978774 100644 --- a/test/fileCatalog.test.ts +++ b/test/fileCatalog.test.ts @@ -19,7 +19,7 @@ describe("app", function () { const databaseConfig = { name: 'geesome_test', options: { logging: () => { - }, storage: 'database-test.sqlite' + }, dialect: 'sqlite', storage: 'database-test.sqlite' } }; diff --git a/test/foreignAccounts.test.ts b/test/foreignAccounts.test.ts index 4da29f4a..a71ccdeb 100644 --- a/test/foreignAccounts.test.ts +++ b/test/foreignAccounts.test.ts @@ -19,7 +19,7 @@ describe("foreignAccounts", function () { const databaseConfig = { name: 'geesome_test', options: { logging: () => { - }, storage: 'database-test.sqlite' + }, dialect: 'sqlite', storage: 'database-test.sqlite' } }; diff --git a/test/group.test.ts b/test/group.test.ts index e153e453..5e31ca20 100644 --- a/test/group.test.ts +++ b/test/group.test.ts @@ -15,11 +15,12 @@ const commonHelpers = require("geesome-libs/src/common"); const assert = require('assert'); const _ = require('lodash'); -describe("group", function () { +describe.only("group", function () { const databaseConfig = { name: 'geesome_test', options: { - logging: () => { - }, storage: 'database-test.sqlite' + logging: () => {}, + dialect: 'sqlite', + storage: 'database-test.sqlite' } }; diff --git a/test/groupCategory.test.ts b/test/groupCategory.test.ts index 01fe383b..76c98313 100644 --- a/test/groupCategory.test.ts +++ b/test/groupCategory.test.ts @@ -24,7 +24,7 @@ describe("groupCategory", function () { const databaseConfig = { name: 'geesome_test', options: { logging: () => { - }, storage: 'database-test.sqlite' + }, dialect: 'sqlite', storage: 'database-test.sqlite' } }; diff --git a/test/invites.test.ts b/test/invites.test.ts index d6b5b9e6..23da415d 100644 --- a/test/invites.test.ts +++ b/test/invites.test.ts @@ -24,7 +24,7 @@ describe("app", function () { const databaseConfig = { name: 'geesome_test', options: { logging: () => { - }, storage: 'database-test.sqlite' + }, dialect: 'sqlite', storage: 'database-test.sqlite' } }; diff --git a/test/pin.test.ts b/test/pin.test.ts index 87466ba2..b4794cf0 100644 --- a/test/pin.test.ts +++ b/test/pin.test.ts @@ -19,7 +19,7 @@ describe("pin", function () { const databaseConfig = { name: 'geesome_test', options: { logging: () => { - }, storage: 'database-test.sqlite' + }, dialect: 'sqlite', storage: 'database-test.sqlite' } }; diff --git a/test/render.test.ts b/test/render.test.ts index 78d86889..dfd179d7 100644 --- a/test/render.test.ts +++ b/test/render.test.ts @@ -24,7 +24,7 @@ describe("renders", function () { const databaseConfig = { name: 'geesome_test', options: { logging: () => { - }, storage: 'database-test.sqlite' + }, dialect: 'sqlite', storage: 'database-test.sqlite' } }; diff --git a/test/staticSiteGenerator.test.ts b/test/staticSiteGenerator.test.ts index aef3ecec..d58564cb 100644 --- a/test/staticSiteGenerator.test.ts +++ b/test/staticSiteGenerator.test.ts @@ -25,7 +25,7 @@ describe("staticSiteGenerator", function () { const databaseConfig = { name: 'geesome_test', options: { logging: () => { - }, storage: 'database-test.sqlite' + }, dialect: 'sqlite', storage: 'database-test.sqlite' } }; @@ -112,7 +112,7 @@ describe("staticSiteGenerator", function () { posts.push(await app.ms.group.createPost(testUser.id, postData)); } - const directoryStorageId = await staticSiteGenerator.generate(testUser.id, 'group', testGroup.id, { + const directoryStorageId = await staticSiteGenerator.generateGroupSite(testUser.id, 'group', testGroup.id, { lang: 'en', dateFormat: 'DD.MM.YYYY hh:mm:ss', baseStorageUri: 'http://localhost:2052/ipfs/', diff --git a/test/storage.test.ts b/test/storage.test.ts index ec250609..686a0696 100644 --- a/test/storage.test.ts +++ b/test/storage.test.ts @@ -20,7 +20,7 @@ describe("storage", function () { const databaseConfig = { name: 'geesome_test', options: { logging: () => { - }, storage: 'database-test.sqlite' + }, dialect: 'sqlite', storage: 'database-test.sqlite' } }; let storage: IGeesomeStorageModule; diff --git a/test/telegramClient.test.ts b/test/telegramClient.test.ts index 2c2293e5..3de9359b 100644 --- a/test/telegramClient.test.ts +++ b/test/telegramClient.test.ts @@ -29,7 +29,7 @@ describe("telegramClient", function () { const databaseConfig = { name: 'geesome_test', options: { logging: () => { - }, storage: 'database-test.sqlite' + }, dialect: 'sqlite', storage: 'database-test.sqlite' } }; diff --git a/test/twitterClient.test.ts b/test/twitterClient.test.ts index 7f644487..0f00c80c 100644 --- a/test/twitterClient.test.ts +++ b/test/twitterClient.test.ts @@ -26,7 +26,7 @@ describe.skip("twitterClient", function () { const databaseConfig = { name: 'geesome_test', options: { logging: () => { - }, storage: 'database-test.sqlite' + }, dialect: 'sqlite', storage: 'database-test.sqlite' } }; From 35fff96956800f5e29b3d1b95eda4818d8197374 Mon Sep 17 00:00:00 2001 From: MicrowaveDev Date: Thu, 10 Oct 2024 05:02:27 +0100 Subject: [PATCH 2/2] add static site manager by content ids --- app/modules/staticSiteGenerator/api.ts | 2 +- app/modules/staticSiteGenerator/index.ts | 166 +++++++++++++----- app/modules/staticSiteGenerator/interface.ts | 4 +- app/modules/staticSiteGenerator/models.ts | 2 +- .../site/public/pages/ContentList/index.js | 36 ++++ .../site/public/pages/EmptyLayout/index.js | 23 +++ 6 files changed, 188 insertions(+), 45 deletions(-) create mode 100644 app/modules/staticSiteGenerator/site/public/pages/ContentList/index.js create mode 100644 app/modules/staticSiteGenerator/site/public/pages/EmptyLayout/index.js diff --git a/app/modules/staticSiteGenerator/api.ts b/app/modules/staticSiteGenerator/api.ts index 7d62a873..85379dc0 100644 --- a/app/modules/staticSiteGenerator/api.ts +++ b/app/modules/staticSiteGenerator/api.ts @@ -15,7 +15,7 @@ module.exports = (_app: IGeesomeApp, ssgModule: IGeesomeStaticSiteGeneratorModul return res.send(await ssgModule.getStaticSiteInfo(req.user.id, req.body.entityType, req.body.entityId), 200); }); api.onAuthorizedPost('run', async (req, res) => { - return res.send(await ssgModule.addRenderToQueueAndProcess(req.user.id, req.apiKey.id, req.body.entityType, req.body.entityId, req.body.options), 200); + return res.send(await ssgModule.addRenderToQueueAndProcess(req.user.id, req.apiKey.id, _.pick(req.body, ['entityType', 'entityId']), req.body.options), 200); }); api.onAuthorizedPost('bind-to-static-id/:id', async (req, res) => { return res.send(await ssgModule.bindSiteToStaticId(req.user.id, req.params.id), 200); diff --git a/app/modules/staticSiteGenerator/index.ts b/app/modules/staticSiteGenerator/index.ts index 3dc13b7b..fb217342 100644 --- a/app/modules/staticSiteGenerator/index.ts +++ b/app/modules/staticSiteGenerator/index.ts @@ -25,12 +25,17 @@ function getModule(app: IGeesomeApp, models, prepareRender) { class StaticSiteGenerator implements IGeesomeStaticSiteGeneratorModule { moduleName = 'static-site-generator'; - async addRenderToQueueAndProcess(userId, userApiKeyId, entityType, entityId, options) { + async addRenderToQueueAndProcess(userId, userApiKeyId, renderData: {entityType, entityId?, entityIds?}, options) { + const {entityType, entityId, entityIds} = renderData; if (entityType === 'group') { const isAdmin = await app.ms.group.isAdminInGroup(userId, entityId); if (!isAdmin) { throw Error('not_enough_rights'); } + } else if (entityType === 'content-list') { + if (!entityIds || !entityIds.length || entityIds.length > 9999) { + throw Error('invalid_entity_ids'); + } } else { throw Error('unknown_type'); } @@ -48,7 +53,7 @@ function getModule(app: IGeesomeApp, models, prepareRender) { } async runRenderAndWaitForFinish(userId, apiKeyId, entityType, entityId, options) { - const operationQueue = await this.addRenderToQueueAndProcess(userId, apiKeyId, entityType, entityId, options); + const operationQueue = await this.addRenderToQueueAndProcess(userId, apiKeyId, {entityType, entityId}, options); const finishedOperation = await new Promise((resolve) => { finishCallbacks[operationQueue.id] = resolve; }); @@ -74,22 +79,36 @@ function getModule(app: IGeesomeApp, models, prepareRender) { } const {userId, userApiKeyId} = waitingQueue; - const {entityType, entityId, options} = JSON.parse(waitingQueue.inputJson); - + const {entityType, entityId, entityIds, options} = JSON.parse(waitingQueue.inputJson); + let operationPrefix; + if (entityType === 'content-list') { + operationPrefix = 'type:' + entityType + ';id:' + helpers.keccak(JSON.stringify(entityIds)); + } else { + operationPrefix = 'type:' + entityType + ';id:' + entityId; + } const asyncOperation = await app.ms.asyncOperation.addAsyncOperation(userId, { userApiKeyId, name: 'run-' + this.moduleName, - channel: 'type:' + entityType + ';id:' + entityId + ';op:' + await commonHelper.random() + channel: operationPrefix + ';op:' + await commonHelper.random() }); // console.log('asyncOperation', asyncOperation); await app.ms.asyncOperation.setAsyncOperationToUserOperationQueue(waitingQueue.id, asyncOperation.id); // run in background - this.generate(userId, entityType, entityId, { - ...options, - asyncOperationId: asyncOperation.id - }).then(async (storageId) => { + let operationPromise; + if (entityType === 'content-list') { + operationPromise = this.generateContentListSite(userId, entityType, entityIds, { + ...options, + asyncOperationId: asyncOperation.id + }); + } else { + operationPromise = this.generateGroupSite(userId, entityType, entityId, { + ...options, + asyncOperationId: asyncOperation.id + }); + } + operationPromise.then(async (storageId) => { await app.ms.asyncOperation.closeUserOperationQueueByAsyncOperationId(asyncOperation.id); await app.ms.asyncOperation.finishAsyncOperation(userId, asyncOperation.id); if (finishCallbacks[waitingQueue.id]) { @@ -107,11 +126,11 @@ function getModule(app: IGeesomeApp, models, prepareRender) { } async getDefaultOptionsByGroupId(userId, groupId) { - return this.getDefaultOptions(await app.ms.group.getLocalGroup(userId, groupId)); + return this.getEntityDefaultOptions('group', await app.ms.group.getLocalGroup(userId, groupId)); } - async getDefaultOptions(group) { - const staticSite = await models.StaticSite.findOne({where: {entityType: 'group', entityId: group.id}}); + async getEntityDefaultOptions(entityType, group) { + const staticSite = await models.StaticSite.findOne({where: {entityType, entityId: group.id.toString()}}); let staticSiteOptions = {}; try { staticSiteOptions = JSON.parse(staticSite.options) @@ -137,10 +156,10 @@ function getModule(app: IGeesomeApp, models, prepareRender) { } } - async getResultOptions(group, options) { + async getGroupResultOptions(group, options) { options = _.clone(options) || {}; options.view = options.view || group.view; - const defaultOptions = await this.getDefaultOptions(group); + const defaultOptions = await this.getEntityDefaultOptions('group', group); const merged = _.merge(defaultOptions, options, { site: { avatarStorageId: group.avatarImage ? group.avatarImage.storageId : null, @@ -156,19 +175,28 @@ function getModule(app: IGeesomeApp, models, prepareRender) { return merged; } - async generate(userId, entityType, entityId, options: any = {}): Promise { - // let {baseStorageUri} = options; - // - // baseStorageUri += 'content/'; + async prepareContentListForRender(userId, entityType, entityIds, options: any = {}) { + const entityId = helpers.keccak(JSON.stringify(entityIds)); - const group = await app.ms.group.getLocalGroup(userId, entityId); - let staticSite = await models.StaticSite.findOne({where: {entityType, entityId}}); - if (!staticSite) { - staticSite = await this.createDbStaticSite({userId, entityType, entityId, title: options.title, name: options.site.name, options: JSON.stringify(options)}); - await this.bindSiteToStaticId(userId, staticSite.id); - staticSite = await models.StaticSite.findOne({where: {id: staticSite.id}}); + let staticSite = await this.getOrCreateStaticSite(userId, entityType, entityId, options.title, options.site.name, options); + const contents = await app.ms.database + .getUserContentListByIds(userId, entityIds) + .then(list => list.map(c => c.toJSON())); + const siteStorageDir = `/${staticSite.staticId}-site`; + + await app.ms.storage.makeDir(siteStorageDir).catch(() => {/*already made*/}); + + return { + staticSite, + siteStorageDir, + renderData: { contents, options: {lang: 'en'} } } - options = await this.getResultOptions(group, options); + } + + async prepareGroupPostsForRender(userId, entityType, entityId, options: any = {}) { + const group = await app.ms.group.getLocalGroup(userId, entityId); + let staticSite = await this.getOrCreateStaticSite(userId, entityType, entityId, options.title, options.site.name, options); + options = await this.getGroupResultOptions(group, options); const {list: groupPosts} = await app.ms.group.getGroupPosts(entityId, {}, { sortBy: 'publishedAt', @@ -195,29 +223,81 @@ function getModule(app: IGeesomeApp, models, prepareRender) { const indexById = {}; posts.forEach((p, i) => { indexById[p.id] = i; - }) + }); + + return { + staticSite, + siteStorageDir, + manifestStorageId: group.manifestStorageId, + renderData: { + options, + posts, + pagesCount, + postsPerPage, + indexById + } + } + } + + async getOrCreateStaticSite(userId, entityType, entityId, title, name, options) { + let staticSite = await models.StaticSite.findOne({where: {entityType, entityId: entityId.toString()}}); + if (!staticSite) { + staticSite = await this.createDbStaticSite({userId, entityType, entityId, title: options.title, name: options.site.name, options: JSON.stringify(options)}); + await this.bindSiteToStaticId(userId, staticSite.id); + staticSite = await models.StaticSite.findOne({where: {id: staticSite.id}}); + } + return staticSite; + } + + async generateGroupSite(userId, entityType, entityId, options: any = {}): Promise { + const { + staticSite, + siteStorageDir, + renderData, + manifestStorageId + } = await this.prepareGroupPostsForRender(userId, entityType, entityId, options); // VueSSR: initialize app - const {renderPage, css} = await prepareRender({posts, pagesCount, postsPerPage, options, indexById}); + const {renderPage, css} = await prepareRender(renderData); const {id: cssStorageId} = await app.ms.storage.saveFileByData(css); await app.ms.storage.copyFileFromId(cssStorageId, `${siteStorageDir}/style.css`); // VueSSR: render main page - await this.renderAndSave(renderPage, options, siteStorageDir, ``, 'main'); /* Example for file write: async renderAndWrite(renderPage, routePath) { const htmlContent = await renderPage(routePath); fs.writeFileSync(routePath + '/index.html', htmlContent); } */ - for (let i = 1; i <= pagesCount - 1; i++) { + await this.renderAndSave(renderPage, options, siteStorageDir, ``, 'main'); + for (let i = 1; i <= renderData.pagesCount - 1; i++) { // VueSSR: render other pages by url await this.renderAndSave(renderPage, options, siteStorageDir, `/page/${i}`, 'page'); } - await pIteration.forEachSeries(posts, (p) => this.renderAndSave(renderPage, options, siteStorageDir, `/post/${p.id}`, 'post', p)); + await pIteration.forEachSeries(renderData.posts, (p) => this.renderAndSave(renderPage, options, siteStorageDir, `/post/${p.id}`, 'post', p)); const storageId = await app.ms.storage.getDirectoryId(siteStorageDir); - const baseData = {storageId, lastEntityManifestStorageId: group.manifestStorageId, options: JSON.stringify(options)}; + const baseData = {storageId, lastEntityManifestStorageId: manifestStorageId, options: JSON.stringify(options)}; + await this.updateDbStaticSite(staticSite.id, baseData); + return storageId; + } + + async generateContentListSite(userId, entityType, entityIds, options: any = {}): Promise { + const { + staticSite, + siteStorageDir, + renderData, + } = await this.prepareContentListForRender(userId, entityType, entityIds, options); + + const {renderPage, css} = await prepareRender(renderData); + const {id: cssStorageId} = await app.ms.storage.saveFileByData(css); + await app.ms.storage.copyFileFromId(cssStorageId, `${siteStorageDir}/style.css`); + + await this.copyContentsToSite(siteStorageDir, renderData.contents); + + await this.renderAndSave(renderPage, options, siteStorageDir, ``, 'main'); + const storageId = await app.ms.storage.getDirectoryId(siteStorageDir); + const baseData = {storageId, options: JSON.stringify(options)}; await this.updateDbStaticSite(staticSite.id, baseData); return storageId; } @@ -227,15 +307,7 @@ function getModule(app: IGeesomeApp, models, prepareRender) { return null; } const contents = await app.ms.group.getPostContentWithUrl('', gp); - await pIteration.forEach(contents, async c => { - await app.ms.storage.fileLs('/ipfs/' + c.storageId).then(r => { - console.log('res fileLs', c.storageId, r); - }).catch(e => { - console.error('err fileLs', c.storageId, e); - }); - await app.ms.storage.copyFileFromId(c.storageId, `${siteStorageDir}/content/${c.storageId}${c.type === 'video' ? '.mp4' : ''}`); - await app.ms.storage.copyFileFromId(c.previewStorageId, `${siteStorageDir}/content/${c.previewStorageId}`); - }); + await this.copyContentsToSite(siteStorageDir, contents); return { id: gp.localId, lang: options.lang, @@ -248,6 +320,18 @@ function getModule(app: IGeesomeApp, models, prepareRender) { } } + async copyContentsToSite(siteStorageDir, contents) { + await pIteration.forEach(contents, async c => { + await app.ms.storage.fileLs('/ipfs/' + c.storageId).then(r => { + console.log('res fileLs', c.storageId, r); + }).catch(e => { + console.error('err fileLs', c.storageId, e); + }); + await app.ms.storage.copyFileFromId(c.storageId, `${siteStorageDir}/content/${c.storageId}${c.type === 'video' ? '.mp4' : ''}`); + await app.ms.storage.copyFileFromId(c.previewStorageId, `${siteStorageDir}/content/${c.previewStorageId}`); + }); + } + async renderAndSave(renderPage, options, storageDir, path, type, p = null) { let pageTitle = ''; if (type === 'main') { @@ -298,7 +382,7 @@ function getModule(app: IGeesomeApp, models, prepareRender) { } async getStaticSiteInfo(userId, entityType, entityId) { - const where: any = {entityType, entityId}; + const where: any = {entityType, entityId: entityId.toString()}; if (entityType === 'group') { if(!(await app.ms.group.canEditGroup(userId, entityId))) { throw new Error("not_permitted"); diff --git a/app/modules/staticSiteGenerator/interface.ts b/app/modules/staticSiteGenerator/interface.ts index 7e5afe03..94b38091 100644 --- a/app/modules/staticSiteGenerator/interface.ts +++ b/app/modules/staticSiteGenerator/interface.ts @@ -12,7 +12,7 @@ export default interface IGeesomeStaticSiteGeneratorModule { site: { title, name, username, description, avatarUrl?, postsCount?, base } }>; - addRenderToQueueAndProcess(userId, apiKeyId, type, id, options): Promise; + addRenderToQueueAndProcess(userId, apiKeyId, renderData: {entityType, entityId?, entityIds?}, options): Promise; bindSiteToStaticId(userId, staticSiteId): Promise; @@ -20,7 +20,7 @@ export default interface IGeesomeStaticSiteGeneratorModule { updateStaticSiteInfo(userId, staticSiteId, updateData): Promise; - generate(userId, entityType, entityId, options: any = {}): Promise; + generateGroupSite(userId, entityType, entityId, options: any = {}): Promise; } export interface IStaticSite { diff --git a/app/modules/staticSiteGenerator/models.ts b/app/modules/staticSiteGenerator/models.ts index f3451429..be1a3095 100644 --- a/app/modules/staticSiteGenerator/models.ts +++ b/app/modules/staticSiteGenerator/models.ts @@ -31,7 +31,7 @@ module.exports = async function () { type: Sequelize.STRING(100) }, entityId: { - type: Sequelize.INTEGER + type: Sequelize.STRING(100) }, lastEntityManifestStorageId: { type: Sequelize.STRING(100) diff --git a/app/modules/staticSiteGenerator/site/public/pages/ContentList/index.js b/app/modules/staticSiteGenerator/site/public/pages/ContentList/index.js new file mode 100644 index 00000000..df5cd857 --- /dev/null +++ b/app/modules/staticSiteGenerator/site/public/pages/ContentList/index.js @@ -0,0 +1,36 @@ +import Layout from "../Layout/index.js"; +import {getRelativeRoot} from "../../helpers.js"; + +export default { + components: { + Layout, + }, + inject: ['store'], + created() { + // console.log('post page', this.$route.params.postId); + }, + computed: { + contents() { + return this.store.contents; + }, + contentRoot() { + return getRelativeRoot(this.$route.path) + 'content/'; + }, + }, + template: ` + + + + ` +} diff --git a/app/modules/staticSiteGenerator/site/public/pages/EmptyLayout/index.js b/app/modules/staticSiteGenerator/site/public/pages/EmptyLayout/index.js new file mode 100644 index 00000000..33bac41f --- /dev/null +++ b/app/modules/staticSiteGenerator/site/public/pages/EmptyLayout/index.js @@ -0,0 +1,23 @@ +export default { + components: { + }, + inject: ['store'], + computed: { + headerHtml() { + return this.store.options.headerHtml; + }, + footerHtml() { + return this.store.options.footerHtml; + }, + }, + mounted() { + }, + template: ` +
+ + + +
+ + ` +}