diff --git a/package.json b/package.json index 4f471636..336afff7 100644 --- a/package.json +++ b/package.json @@ -1,18 +1,27 @@ { "author": "Florian Keller ", + "bin": { + "schemastore-updater": "dist/cli.js" + }, "dependencies": { "@types/fs-extra": "5.0.4", "@types/rimraf": "2.0.2", + "commander": "2.16.0", "fs-extra": "7.0.0", "json-schema-to-typescript": "5.5.0", - "program": "1.0.0", "rimraf": "2.6.2", "simple-git": "1.96.0", "typescript": "2.9.2" }, "devDependencies": { + "cross-env": "5.2.0", + "logdown": "3.2.3", "prettier": "1.13.7" }, + "files": [ + "src", + "settings.json" + ], "license": "MIT", "main": "index.js", "name": "schemastore-updater", @@ -22,7 +31,7 @@ "clear": "rimraf dist", "dist": "yarn clear && yarn build", "prettier": "prettier --single-quote --write src/*.ts", - "start": "node dist/index.js" + "start": "cross-env NODE_DEBUG='schemastore-updater/*' node dist/cli.js" }, "version": "1.0.0" } diff --git a/json-schemas.lock.json b/schemas/json-schemas.lock similarity index 100% rename from json-schemas.lock.json rename to schemas/json-schemas.lock diff --git a/settings.json b/settings.json index 3bbcf0cd..ab4cfcc5 100644 --- a/settings.json +++ b/settings.json @@ -39,8 +39,6 @@ "vega-lite.json", "vega.json" ], - "lockFile": "json-schemas.lock.json", - "schemaStoreDir": "./temp/schemastore", - "schemaStoreRepo": "https://github.com/SchemaStore/schemastore", - "tempDir": "./temp" + "lockFile": "schemas/json-schemas.lock", + "schemaStoreRepo": "https://github.com/SchemaStore/schemastore" } diff --git a/src/cli.ts b/src/cli.ts new file mode 100644 index 00000000..3effec9a --- /dev/null +++ b/src/cli.ts @@ -0,0 +1,36 @@ +import { SchemaGenerator } from './'; +import * as program from 'commander'; +import * as path from 'path'; + +const {description, name, version}: {description: string, name: string, version: string} = require('../package.json'); + +program + .name(name) + .version(version) + .description(description) + .option( + '-s, --settings ', + 'Specify a settings file', + path.resolve(__dirname, '..', 'settings.json') + ) + .parse(process.argv); + +const settingsFile = path.resolve(program.settings); + +const { + disabledSchemas, + schemaStoreRepo +}: { + disabledSchemas: string[]; + schemaStoreRepo: string; +} = require(settingsFile); + +const generator = new SchemaGenerator( + disabledSchemas, + schemaStoreRepo +); + +generator + .start() + .then(() => console.log('Done.')) + .catch(error => console.error(error)); diff --git a/src/index.ts b/src/index.ts index 5dd9077a..702bb213 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,40 +5,55 @@ import * as simpleGit from 'simple-git/promise'; import * as fs from 'fs-extra'; import * as crypto from 'crypto'; import * as schemaGenerator from 'json-schema-to-typescript'; +const logdown = require('logdown'); interface SchemaHashes { [fileName: string]: string; } -const git = simpleGit('.'); -const { - disabledSchemas, - lockFile, - schemaStoreDir, - schemaStoreRepo -}: { - disabledSchemas: string; - lockFile: string; - schemaStoreDir: string; - schemaStoreRepo: string; -} = require('../settings.json'); -const schemaStoreDirResolved = path.resolve(schemaStoreDir); -const jsonSchemasDir = path.join(schemaStoreDir, 'src', 'schemas', 'json'); -const lockFileResolved = path.resolve(lockFile); -function generateLockFile(fileName: string, data: SchemaHashes) { - return fs.writeFile(path.resolve(fileName), JSON.stringify(data, null, 2), { - encoding: 'utf8' - }); -} +class SchemaGenerator { + private git: simpleGit.SimpleGit; + private jsonSchemasDir: string; + private lockFile: string; + private schemaStoreDirResolved: string; + private logger: any; + + constructor( + private disabledSchemas: string[], + private schemaStoreRepo: string + ) { + this.git = simpleGit('.'); + this.schemaStoreDirResolved = path.join('temp', 'schemastore'); + this.jsonSchemasDir = path.join(this.schemaStoreDirResolved, 'src', 'schemas', 'json'); + this.lockFile = path.join('schemas', 'json-schemas.lock'); + this.logger = logdown('schemastore-updater', { + logger: console, + markdown: false, + }); + this.logger.info(`Using lockfile "${this.lockFile}".`) + } + + private generateLockFile(fileName: string, data: SchemaHashes) { + return fs.writeFile(path.resolve(fileName), JSON.stringify(data, null, 2), { + encoding: 'utf8' + }); + } -async function generateSchemas(jsonData: SchemaHashes) { - const disabled = []; - for (const fileName in jsonData) { - const fileNameResolved = path.resolve(jsonSchemasDir, fileName); - console.log({ fileName }); - try { - const newSchema = await schemaGenerator.compileFromFile(fileNameResolved); + private async generateSchemas(jsonData: SchemaHashes) { + const disabledSchemas = []; + for (const fileName in jsonData) { + const fileNameResolved = path.resolve(this.jsonSchemasDir, fileName); + this.logger.info(`Processing "${fileName}" ...`); + let newSchema = ''; + try { + newSchema = await schemaGenerator.compileFromFile( + fileNameResolved + ); + } catch (error) { + disabledSchemas.push(fileName); + break; + } const schemaDirResolved = path.resolve( 'schemas', fileName.replace('.json', '') @@ -49,52 +64,55 @@ async function generateSchemas(jsonData: SchemaHashes) { newSchema, { encoding: 'utf8' } ); - } catch (error) { - disabled.push(fileName); } + return { disabledSchemas }; } - console.log({ disabled }); -} -async function start() { - await promisify(rimraf)(schemaStoreDirResolved); - await git.clone(schemaStoreRepo, schemaStoreDirResolved, ['--depth=1']); + async start() { + await promisify(rimraf)(this.schemaStoreDirResolved); + await this.git.clone(this.schemaStoreRepo, this.schemaStoreDirResolved, [ + '--depth=1' + ]); - const jsonFiles = (await fs.readdir(jsonSchemasDir)).filter( - fileName => - fileName.endsWith('.json') && !disabledSchemas.includes(fileName) - ); - const jsonData: SchemaHashes = {}; - for (const fileName of jsonFiles) { - const fileNameResolved = path.resolve(jsonSchemasDir, fileName); - const fileContent = await fs.readFile(fileNameResolved, { - encoding: 'utf8' - }); - const sha256 = crypto - .createHash('sha256') - .update(fileContent) - .digest('hex'); + const jsonFiles = (await fs.readdir(this.jsonSchemasDir)).filter( + fileName => + fileName.endsWith('.json') && !this.disabledSchemas.includes(fileName) + ); + this.logger.info(`Loaded ${jsonFiles.length} schemas.`); + const jsonData: SchemaHashes = {}; + for (const fileName of jsonFiles) { + const fileNameResolved = path.resolve(this.jsonSchemasDir, fileName); + const fileContent = await fs.readFile(fileNameResolved, { + encoding: 'utf8' + }); + const sha256 = crypto + .createHash('sha256') + .update(fileContent) + .digest('hex'); - jsonData[fileName] = sha256; - } + jsonData[fileName] = sha256; + } - if (!fs.existsSync(lockFileResolved)) { - await generateSchemas(jsonData); - await generateLockFile(lockFileResolved, jsonData); - } else { - const lockFileData = await fs.readFile(lockFile, { encoding: 'utf8' }); - const lockFileParsed = JSON.parse(lockFileData); - const failedHashes: SchemaHashes = {}; - for (const fileName in jsonData) { - if ( - !lockFileParsed[fileName] || - jsonData[fileName] !== lockFileParsed[fileName] - ) { - failedHashes[fileName] = jsonData[fileName]; + if (!fs.existsSync(this.lockFile)) { + this.logger.info(`No lockfile exists yet. Writing lockfile to "${this.lockFile}" ..`) + await this.generateSchemas(jsonData); + await this.generateLockFile(this.lockFile, jsonData); + } else { + const lockFileData = await fs.readFile(this.lockFile, { encoding: 'utf8' }); + const lockFileParsed = JSON.parse(lockFileData); + const updatedHashes: SchemaHashes = {}; + for (const fileName in jsonData) { + if ( + !lockFileParsed[fileName] || + jsonData[fileName] !== lockFileParsed[fileName] + ) { + this.logger.info(`Hash from "${fileName}" is outdated. Updating.`) + updatedHashes[fileName] = jsonData[fileName]; + } } + await this.generateSchemas(updatedHashes); } - await generateSchemas(failedHashes); } } -start(); +export { SchemaGenerator }; diff --git a/yarn.lock b/yarn.lock index a49c24ba..b41f081b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -71,6 +71,12 @@ ansi-regex@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df" +ansi-styles@^3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" + dependencies: + color-convert "^1.9.0" + any-promise@^1.0.0: version "1.3.0" resolved "https://registry.yarnpkg.com/any-promise/-/any-promise-1.3.0.tgz#abc6afeedcea52e809cdc0376aed3ce39635d17f" @@ -81,10 +87,6 @@ argparse@^1.0.7: dependencies: sprintf-js "~1.0.2" -async@~0.2.6: - version "0.2.10" - resolved "https://registry.yarnpkg.com/async/-/async-0.2.10.tgz#b6bbe0b0674b9d719708ca38de8c237cb526c3d1" - balanced-match@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" @@ -100,6 +102,14 @@ call-me-maybe@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/call-me-maybe/-/call-me-maybe-1.0.1.tgz#26d208ea89e37b5cbde60250a15f031c16a4d66b" +chalk@^2.3.0: + version "2.4.1" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.1.tgz#18c49ab16a037b6eb0152cc83e3471338215b66e" + dependencies: + ansi-styles "^3.2.1" + escape-string-regexp "^1.0.5" + supports-color "^5.3.0" + cli-color@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/cli-color/-/cli-color-1.2.0.tgz#3a5ae74fd76b6267af666e69e2afbbd01def34d1" @@ -111,10 +121,41 @@ cli-color@^1.2.0: memoizee "^0.4.3" timers-ext "0.1" +color-convert@^1.9.0: + version "1.9.2" + resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.2.tgz#49881b8fba67df12a96bdf3f56c0aab9e7913147" + dependencies: + color-name "1.1.1" + +color-name@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.1.tgz#4b1415304cf50028ea81643643bd82ea05803689" + +commander@2.16.0: + version "2.16.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-2.16.0.tgz#f16390593996ceb4f3eeb020b31d78528f7f8a50" + concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" +cross-env@5.2.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/cross-env/-/cross-env-5.2.0.tgz#6ecd4c015d5773e614039ee529076669b9d126f2" + dependencies: + cross-spawn "^6.0.5" + is-windows "^1.0.0" + +cross-spawn@^6.0.5: + version "6.0.5" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4" + dependencies: + nice-try "^1.0.4" + path-key "^2.0.1" + semver "^5.5.0" + shebang-command "^1.2.0" + which "^1.2.9" + d@1: version "1.0.0" resolved "https://registry.yarnpkg.com/d/-/d-1.0.0.tgz#754bb5bfe55451da69a58b94d45f4c5b0462d58f" @@ -159,6 +200,10 @@ es6-weak-map@^2.0.2: es6-iterator "^2.0.1" es6-symbol "^3.1.1" +escape-string-regexp@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" + esprima@^4.0.0: version "4.0.1" resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71" @@ -170,12 +215,6 @@ event-emitter@^0.3.5: d "1" es5-ext "~0.10.14" -expand-tilde@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/expand-tilde/-/expand-tilde-2.0.2.tgz#97e801aa052df02454de46b02bf621642cdc8502" - dependencies: - homedir-polyfill "^1.0.1" - format-util@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/format-util/-/format-util-1.0.3.tgz#032dca4a116262a12c43f4c3ec8566416c5b2d95" @@ -207,11 +246,9 @@ graceful-fs@^4.1.2, graceful-fs@^4.1.6: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" -homedir-polyfill@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/homedir-polyfill/-/homedir-polyfill-1.0.1.tgz#4c2bbc8a758998feebf5ed68580f76d46768b4bc" - dependencies: - parse-passwd "^1.0.0" +has-flag@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" inflight@^1.0.4: version "1.0.6" @@ -224,14 +261,18 @@ inherits@2: version "2.0.3" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" -ini@~1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/ini/-/ini-1.1.0.tgz#4e808c2ce144c6c1788918e034d6797bc6cf6281" - is-promise@^2.1: version "2.1.0" resolved "https://registry.yarnpkg.com/is-promise/-/is-promise-2.1.0.tgz#79a2a9ece7f096e80f36d2b2f3bc16c1ff4bf3fa" +is-windows@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/is-windows/-/is-windows-1.0.2.tgz#d1850eb9791ecd18e6182ce12a30f396634bb19d" + +isexe@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" + js-yaml@^3.10.0: version "3.12.0" resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.12.0.tgz#eaed656ec8344f10f527c6bfa1b6e2244de167d1" @@ -282,6 +323,12 @@ lodash@^4.17.4: version "4.17.10" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.10.tgz#1b7793cf7259ea38fb3661d4d38b3260af8ae4e7" +logdown@3.2.3: + version "3.2.3" + resolved "https://registry.yarnpkg.com/logdown/-/logdown-3.2.3.tgz#b033901259057a419cb09a41baa3e98b5be57b3c" + dependencies: + chalk "^2.3.0" + lru-queue@0.1: version "0.1.0" resolved "https://registry.yarnpkg.com/lru-queue/-/lru-queue-0.1.0.tgz#2738bd9f0d3cf4f84490c5736c48699ac632cda3" @@ -327,6 +374,10 @@ next-tick@1: version "1.0.0" resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.0.0.tgz#ca86d1fe8828169b0120208e3dc8424b9db8342c" +nice-try@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.4.tgz#d93962f6c52f2c1558c0fbda6d512819f1efe1c4" + object-assign@^4.0.1: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" @@ -343,32 +394,38 @@ ono@^4.0.3: dependencies: format-util "^1.0.3" -parse-passwd@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/parse-passwd/-/parse-passwd-1.0.0.tgz#6d5b934a456993b23d37f40a382d6f1666a8e5c6" - path-is-absolute@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" +path-key@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/path-key/-/path-key-2.0.1.tgz#411cadb574c5a140d3a4b1910d40d80cc9f40b40" + prettier@1.13.7, prettier@^1.8.2: version "1.13.7" resolved "https://registry.yarnpkg.com/prettier/-/prettier-1.13.7.tgz#850f3b8af784a49a6ea2d2eaa7ed1428a34b7281" -program@1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/program/-/program-1.0.0.tgz#5641f4384c6c8bb6bf0f00dc458f728f24ad14a4" - dependencies: - async "~0.2.6" - expand-tilde "^2.0.2" - ini "~1.1.0" - rimraf@2.6.2: version "2.6.2" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.2.tgz#2ed8150d24a16ea8651e6d6ef0f47c4158ce7a36" dependencies: glob "^7.0.5" +semver@^5.5.0: + version "5.5.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.5.0.tgz#dc4bbc7a6ca9d916dee5d43516f0092b58f7b8ab" + +shebang-command@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-1.2.0.tgz#44aac65b695b03398968c39f363fee5deafdf1ea" + dependencies: + shebang-regex "^1.0.0" + +shebang-regex@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-1.0.0.tgz#da42f49740c0b42db2ca9728571cb190c98efea3" + simple-git@1.96.0: version "1.96.0" resolved "https://registry.yarnpkg.com/simple-git/-/simple-git-1.96.0.tgz#d0279b771ed89a2d1e7d9e15f9abd7f26e319d7c" @@ -383,6 +440,12 @@ stdin@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/stdin/-/stdin-0.0.1.tgz#d3041981aaec3dfdbc77a1b38d6372e38f5fb71e" +supports-color@^5.3.0: + version "5.4.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.4.0.tgz#1c6b337402c2137605efe19f10fec390f6faab54" + dependencies: + has-flag "^3.0.0" + thenify-all@^1.0.0: version "1.6.0" resolved "https://registry.yarnpkg.com/thenify-all/-/thenify-all-1.6.0.tgz#1a1918d402d8fc3f98fbf234db0bcc8cc10e9726" @@ -410,6 +473,12 @@ universalify@^0.1.0: version "0.1.2" resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66" +which@^1.2.9: + version "1.3.1" + resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a" + dependencies: + isexe "^2.0.0" + wrappy@1: version "1.0.2" resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"