From 6f913b7e9f0ef9d0ba4070fb935ff145d6638c60 Mon Sep 17 00:00:00 2001 From: Sonal Saldanha Date: Thu, 13 Jul 2017 15:27:24 +0530 Subject: [PATCH 1/2] Add new plugin to handle csv files --- packages/gatsby-transformer-csv/README.md | 98 +++++++- packages/gatsby-transformer-csv/package.json | 15 +- packages/gatsby-transformer-csv/src/.gitkeep | 0 .../__snapshots__/gatsby-node.js.snap | 231 ++++++++++++++++++ .../src/__tests__/gatsby-node.js | 114 +++++++++ .../gatsby-transformer-csv/src/gatsby-node.js | 62 +++++ 6 files changed, 515 insertions(+), 5 deletions(-) delete mode 100644 packages/gatsby-transformer-csv/src/.gitkeep create mode 100644 packages/gatsby-transformer-csv/src/__tests__/__snapshots__/gatsby-node.js.snap create mode 100644 packages/gatsby-transformer-csv/src/__tests__/gatsby-node.js create mode 100644 packages/gatsby-transformer-csv/src/gatsby-node.js diff --git a/packages/gatsby-transformer-csv/README.md b/packages/gatsby-transformer-csv/README.md index 3a17fedaea23d..b83aa304f8b0c 100644 --- a/packages/gatsby-transformer-csv/README.md +++ b/packages/gatsby-transformer-csv/README.md @@ -1,3 +1,99 @@ # gatsby-transformer-csv -Stub README +Parses CSV files into JSON arrays. + +## Install + +`npm install --save gatsby-transformer-csv` + +## How to use + +```javascript +// In your gatsby-config.js +plugins: [ + `gatsby-transformer-csv`, +] +``` +Above is the minimal configuration required to begin working. Additional customization +of the parsing process is possible using the parameters listed in +[csvtojson](https://github.com/Keyang/node-csvtojson#parameters). + +```javascript +// In your gatsby-config.js +plugins: [ + { + resolve: `gatsby-transformer-csv`, + options: { + noheader: true + } + } +] +``` + +## Parsing algorithm + +Each row is converted into a node with CSV headers as the keys. + +So if your project has a `letters.csv` with +``` +letter,value +a,65 +b,66 +c,67 +``` +the following three nodes would be created. + +```javascript +[ + { letter: 'a', value: 65, type: 'LettersCsv' }, + { letter: 'b', value: 66, type: 'LettersCsv' }, + { letter: 'c', value: 67, type: 'LettersCsv' }, +] +``` + +## How to query + +You'd be able to query your letters like: + +```graphql +{ + allLettersCsv { + edges { + node { + letter + value + } + } + } +} +``` + +Which would return: + +```javascript +{ + allLettersCsv: { + edges: [ + { + node: { + letter: 'a' + value: 65 + } + }, + { + node: { + letter: 'b' + value: 66 + } + }, + { + node: { + letter: 'c' + value: 67 + } + } + ] + } +} +``` + diff --git a/packages/gatsby-transformer-csv/package.json b/packages/gatsby-transformer-csv/package.json index f387e93c5dd94..be4254fba0fed 100644 --- a/packages/gatsby-transformer-csv/package.json +++ b/packages/gatsby-transformer-csv/package.json @@ -1,18 +1,25 @@ { "name": "gatsby-transformer-csv", "version": "1.0.1", - "description": "Stub description for gatsby-transformer-csv", + "description": "Gatsby transformer plugin for CSV files", "main": "index.js", "scripts": { "build": "babel src --out-dir . --ignore __tests__", "watch": "babel -w src --out-dir . --ignore __tests__" }, "keywords": [ - "gatsby" + "gatsby", + "csv", + "gatsby-plugin" ], - "author": "Kyle Mathews <mathews.kyle@gmail.com>", + "author": "Sonal Saldanha ", "license": "MIT", "devDependencies": { - "babel-cli": "^6.24.1" + "babel-cli": "^6.24.1", + "json2csv":"^3.7" + }, + "dependencies": { + "bluebird": "^3.5.0", + "csvtojson": "^1.1" } } diff --git a/packages/gatsby-transformer-csv/src/.gitkeep b/packages/gatsby-transformer-csv/src/.gitkeep deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/packages/gatsby-transformer-csv/src/__tests__/__snapshots__/gatsby-node.js.snap b/packages/gatsby-transformer-csv/src/__tests__/__snapshots__/gatsby-node.js.snap new file mode 100644 index 0000000000000..7543c0171ce5f --- /dev/null +++ b/packages/gatsby-transformer-csv/src/__tests__/__snapshots__/gatsby-node.js.snap @@ -0,0 +1,231 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Process nodes correctly correctly creates nodes from JSON which is an array of objects 1`] = ` +Array [ + Array [ + Object { + "blue": "true", + "children": Array [], + "funny": "yup", + "id": "whatever [0] >>> CSV", + "internal": Object { + "contentDigest": "5005aee6b2557974cae0aabc712e125a", + "type": "TestCsv", + }, + "parent": "whatever", + }, + ], + Array [ + Object { + "blue": "false", + "children": Array [], + "funny": "nope", + "id": "whatever [1] >>> CSV", + "internal": Object { + "contentDigest": "0af87da7e572c2d6d8493bcd642c2c78", + "type": "TestCsv", + }, + "parent": "whatever", + }, + ], +] +`; + +exports[`Process nodes correctly correctly creates nodes from JSON which is an array of objects 2`] = ` +Array [ + Array [ + Object { + "child": Object { + "blue": "true", + "children": Array [], + "funny": "yup", + "id": "whatever [0] >>> CSV", + "internal": Object { + "contentDigest": "5005aee6b2557974cae0aabc712e125a", + "type": "TestCsv", + }, + "parent": "whatever", + }, + "parent": Object { + "children": Array [], + "content": "\\"blue\\",\\"funny\\" +true,\\"yup\\" +false,\\"nope\\"", + "extension": "csv", + "id": "whatever", + "internal": Object { + "contentDigest": "whatever", + "mediaType": "text/csv", + }, + "name": "test", + "parent": "SOURCE", + }, + }, + ], + Array [ + Object { + "child": Object { + "blue": "false", + "children": Array [], + "funny": "nope", + "id": "whatever [1] >>> CSV", + "internal": Object { + "contentDigest": "0af87da7e572c2d6d8493bcd642c2c78", + "type": "TestCsv", + }, + "parent": "whatever", + }, + "parent": Object { + "children": Array [], + "content": "\\"blue\\",\\"funny\\" +true,\\"yup\\" +false,\\"nope\\"", + "extension": "csv", + "id": "whatever", + "internal": Object { + "contentDigest": "whatever", + "mediaType": "text/csv", + }, + "name": "test", + "parent": "SOURCE", + }, + }, + ], +] +`; + +exports[`Process nodes correctly correctly handles the options object that is passed to it 1`] = ` +Array [ + Array [ + Object { + "children": Array [], + "field1": "blue", + "field2": "funny", + "id": "whatever [0] >>> CSV", + "internal": Object { + "contentDigest": "49605f5e32a2250d7ca740c5d2e79784", + "type": "TestCsv", + }, + "parent": "whatever", + }, + ], + Array [ + Object { + "children": Array [], + "field1": "true", + "field2": "yup", + "id": "whatever [1] >>> CSV", + "internal": Object { + "contentDigest": "2bf0bc0e8539ef5f6bbcf67c8b733233", + "type": "TestCsv", + }, + "parent": "whatever", + }, + ], + Array [ + Object { + "children": Array [], + "field1": "false", + "field2": "nope", + "id": "whatever [2] >>> CSV", + "internal": Object { + "contentDigest": "f5f617ce4243d639de924543dec5600e", + "type": "TestCsv", + }, + "parent": "whatever", + }, + ], +] +`; + +exports[`Process nodes correctly correctly handles the options object that is passed to it 2`] = ` +Array [ + Array [ + Object { + "child": Object { + "children": Array [], + "field1": "blue", + "field2": "funny", + "id": "whatever [0] >>> CSV", + "internal": Object { + "contentDigest": "49605f5e32a2250d7ca740c5d2e79784", + "type": "TestCsv", + }, + "parent": "whatever", + }, + "parent": Object { + "children": Array [], + "content": "blue,funny +true,yup +false,nope", + "extension": "csv", + "id": "whatever", + "internal": Object { + "contentDigest": "whatever", + "mediaType": "text/csv", + }, + "name": "test", + "parent": "SOURCE", + }, + }, + ], + Array [ + Object { + "child": Object { + "children": Array [], + "field1": "true", + "field2": "yup", + "id": "whatever [1] >>> CSV", + "internal": Object { + "contentDigest": "2bf0bc0e8539ef5f6bbcf67c8b733233", + "type": "TestCsv", + }, + "parent": "whatever", + }, + "parent": Object { + "children": Array [], + "content": "blue,funny +true,yup +false,nope", + "extension": "csv", + "id": "whatever", + "internal": Object { + "contentDigest": "whatever", + "mediaType": "text/csv", + }, + "name": "test", + "parent": "SOURCE", + }, + }, + ], + Array [ + Object { + "child": Object { + "children": Array [], + "field1": "false", + "field2": "nope", + "id": "whatever [2] >>> CSV", + "internal": Object { + "contentDigest": "f5f617ce4243d639de924543dec5600e", + "type": "TestCsv", + }, + "parent": "whatever", + }, + "parent": Object { + "children": Array [], + "content": "blue,funny +true,yup +false,nope", + "extension": "csv", + "id": "whatever", + "internal": Object { + "contentDigest": "whatever", + "mediaType": "text/csv", + }, + "name": "test", + "parent": "SOURCE", + }, + }, + ], +] +`; diff --git a/packages/gatsby-transformer-csv/src/__tests__/gatsby-node.js b/packages/gatsby-transformer-csv/src/__tests__/gatsby-node.js new file mode 100644 index 0000000000000..02b092d0b498e --- /dev/null +++ b/packages/gatsby-transformer-csv/src/__tests__/gatsby-node.js @@ -0,0 +1,114 @@ +const Promise = require(`bluebird`) +const _ = require(`lodash`) +const json2csv = require(`json2csv`) + +const { onCreateNode } = require(`../gatsby-node`) + +describe(`Process nodes correctly`, () => { + const node = { + id: `whatever`, + parent: `SOURCE`, + children: [], + extension: `csv`, + internal: { + contentDigest: `whatever`, + mediaType: `text/csv`, + }, + name: `test`, + } + + // Make some fake functions its expecting. + const loadNodeContent = node => Promise.resolve(node.content) + + it(`correctly creates nodes from JSON which is an array of objects`, async () => { + const fields = [`blue`, `funny`] + const data = [{ blue: true, funny: `yup` }, { blue: false, funny: `nope` }] + const csv = json2csv({ data: data, fields: fields }) + node.content = csv + + const createNode = jest.fn() + const createParentChildLink = jest.fn() + const boundActionCreators = { createNode, createParentChildLink } + + await onCreateNode({ + node, + loadNodeContent, + boundActionCreators, + }).then(() => { + expect(createNode.mock.calls).toMatchSnapshot() + expect(createParentChildLink.mock.calls).toMatchSnapshot() + expect(createNode).toHaveBeenCalledTimes(2) + expect(createParentChildLink).toHaveBeenCalledTimes(2) + }) + }) + + it(`correctly handles the options object that is passed to it`, async () => { + node.content = `blue,funny\ntrue,yup\nfalse,nope` + + const createNode = jest.fn() + const createParentChildLink = jest.fn() + const boundActionCreators = { createNode, createParentChildLink } + + await onCreateNode( + { + node, + loadNodeContent, + boundActionCreators, + }, + { noheader: true } + ).then(() => { + expect(createNode.mock.calls).toMatchSnapshot() + expect(createParentChildLink.mock.calls).toMatchSnapshot() + expect(createNode).toHaveBeenCalledTimes(3) + expect(createParentChildLink).toHaveBeenCalledTimes(3) + }) + }) + + it(`If the object has an id, it uses that as the id instead of the auto-generated one`, async () => { + const fields = [`id`, `blue`, `funny`] + const data = [ + { id: `foo`, blue: true, funny: `yup` }, + { blue: false, funny: `nope` }, + ] + const csv = json2csv({ data: data, fields: fields }) + node.content = csv + + const createNode = jest.fn() + const createParentChildLink = jest.fn() + const boundActionCreators = { createNode, createParentChildLink } + + await onCreateNode({ + node, + loadNodeContent, + boundActionCreators, + }).then(() => { + expect(createNode.mock.calls[0][0].id).toEqual(`foo`) + }) + }) + + it(`the different objects shouldn't get the same ID even if they have the same content`, async () => { + const fields = [`id`, `blue`, `funny`] + const data = [ + { id: `foo`, blue: true, funny: `yup` }, + { blue: false, funny: `nope` }, + { blue: false, funny: `nope` }, + { green: false, funny: `nope` }, + ] + const csv = json2csv({ data: data, fields: fields }) + node.content = csv + + const createNode = jest.fn() + const createParentChildLink = jest.fn() + const boundActionCreators = { createNode, createParentChildLink } + + await onCreateNode({ + node, + loadNodeContent, + boundActionCreators, + }).then(() => { + const ids = createNode.mock.calls.map(object => object[0].id) + // Test that they're unique + expect(_.uniq(ids).length).toEqual(4) + }) + }) +}) diff --git a/packages/gatsby-transformer-csv/src/gatsby-node.js b/packages/gatsby-transformer-csv/src/gatsby-node.js new file mode 100644 index 0000000000000..0c21db54f40d1 --- /dev/null +++ b/packages/gatsby-transformer-csv/src/gatsby-node.js @@ -0,0 +1,62 @@ +const Promise = require(`bluebird`) +const csv = require(`csvtojson`) +const _ = require(`lodash`) +const crypto = require(`crypto`) + +const convertToJson = (data, options) => + new Promise((res, rej) => { + csv(options).fromString(data).on(`end_parsed`, jsonData => { + if (!jsonData) { + rej(`CSV to JSON conversion failed!`) + } + res(jsonData) + }) + }) + +async function onCreateNode( + { node, boundActionCreators, loadNodeContent }, + options +) { + const { createNode, createParentChildLink } = boundActionCreators + // Filter out non-csv content + if (node.extension !== `csv`) { + return + } + // Load CSV contents + const content = await loadNodeContent(node) + // Parse + let parsedContent = await convertToJson(content, options) + + if (_.isArray(parsedContent)) { + const csvArray = parsedContent.map((obj, i) => { + const objStr = JSON.stringify(obj) + const contentDigest = crypto + .createHash(`md5`) + .update(objStr) + .digest(`hex`) + + return { + ...obj, + id: obj.id ? obj.id : `${node.id} [${i}] >>> CSV`, + children: [], + parent: node.id, + internal: { + contentDigest, + // TODO make choosing the "type" a lot smarter. This assumes + // the parent node is a file. + // PascalCase + type: _.upperFirst(_.camelCase(`${node.name} Csv`)), + }, + } + }) + + _.each(csvArray, y => { + createNode(y) + createParentChildLink({ parent: node, child: y }) + }) + } + + return +} + +exports.onCreateNode = onCreateNode From 944429b93f9dbf21783efdfce90b61d6ff94d248 Mon Sep 17 00:00:00 2001 From: Sonal Saldanha Date: Fri, 14 Jul 2017 13:40:56 +0530 Subject: [PATCH 2/2] Add csv usage example --- examples/using-csv/.eslintrc | 8 +++++ examples/using-csv/.gitignore | 3 ++ examples/using-csv/README.md | 3 ++ examples/using-csv/gatsby-config.js | 22 +++++++++++++ examples/using-csv/package.json | 19 +++++++++++ examples/using-csv/src/data/letters.csv | 27 +++++++++++++++ examples/using-csv/src/pages/index.js | 44 +++++++++++++++++++++++++ 7 files changed, 126 insertions(+) create mode 100644 examples/using-csv/.eslintrc create mode 100644 examples/using-csv/.gitignore create mode 100644 examples/using-csv/README.md create mode 100644 examples/using-csv/gatsby-config.js create mode 100644 examples/using-csv/package.json create mode 100644 examples/using-csv/src/data/letters.csv create mode 100644 examples/using-csv/src/pages/index.js diff --git a/examples/using-csv/.eslintrc b/examples/using-csv/.eslintrc new file mode 100644 index 0000000000000..aadde9c0aa03d --- /dev/null +++ b/examples/using-csv/.eslintrc @@ -0,0 +1,8 @@ +{ + "env": { + "browser": true + }, + "globals": { + "graphql": false + } +} \ No newline at end of file diff --git a/examples/using-csv/.gitignore b/examples/using-csv/.gitignore new file mode 100644 index 0000000000000..8f5b35a4a9cbc --- /dev/null +++ b/examples/using-csv/.gitignore @@ -0,0 +1,3 @@ +public +.cache +node_modules diff --git a/examples/using-csv/README.md b/examples/using-csv/README.md new file mode 100644 index 0000000000000..7d0bcfebd9ea0 --- /dev/null +++ b/examples/using-csv/README.md @@ -0,0 +1,3 @@ +# using-csv + +https://using-csv.gatsbyjs.org diff --git a/examples/using-csv/gatsby-config.js b/examples/using-csv/gatsby-config.js new file mode 100644 index 0000000000000..75ee35c79ff02 --- /dev/null +++ b/examples/using-csv/gatsby-config.js @@ -0,0 +1,22 @@ +module.exports = { + siteMetadata: { + title: `gatsby-example-using-csv`, + description: `Blazing-fast React.js static site generator`, + }, + plugins: [ + `gatsby-transformer-csv`, + { + resolve: `gatsby-plugin-google-analytics`, + options: { + trackingId: `UA-93349937-2`, + }, + }, + { + resolve: `gatsby-source-filesystem`, + options: { + path: `${__dirname}/src/data`, + name: `data`, + }, + }, + ], +} diff --git a/examples/using-csv/package.json b/examples/using-csv/package.json new file mode 100644 index 0000000000000..c1d2d080cd1d8 --- /dev/null +++ b/examples/using-csv/package.json @@ -0,0 +1,19 @@ +{ + "name": "using-csv", + "private": true, + "description": "Gatsby example site using using-csv", + "author": "Sonal Saldanha ", + "dependencies": { + "gatsby": "latest", + "gatsby-link": "latest", + "gatsby-plugin-google-analytics": "latest", + "gatsby-source-filesystem": "latest", + "gatsby-transformer-csv": "latest" + }, + "license": "MIT", + "main": "n/a", + "scripts": { + "develop": "gatsby develop", + "build": "gatsby build" + } +} diff --git a/examples/using-csv/src/data/letters.csv b/examples/using-csv/src/data/letters.csv new file mode 100644 index 0000000000000..2c5fa50429326 --- /dev/null +++ b/examples/using-csv/src/data/letters.csv @@ -0,0 +1,27 @@ +letter,value +A,65 +B,66 +C,67 +D,68 +E,69 +F,70 +G,71 +H,72 +I,73 +J,74 +K,75 +L,76 +M,77 +N,78 +O,79 +P,80 +Q,81 +R,82 +S,83 +T,84 +U,85 +V,86 +W,87 +X,88 +Y,89 +Z,90 diff --git a/examples/using-csv/src/pages/index.js b/examples/using-csv/src/pages/index.js new file mode 100644 index 0000000000000..cbb4e7941d8d6 --- /dev/null +++ b/examples/using-csv/src/pages/index.js @@ -0,0 +1,44 @@ +import React from "react" + +class IndexComponent extends React.Component { + render() { + const data = this.props.data.allLettersCsv.edges + return ( +
+ + + + + + + + + {data.map((row,i) => ( + + + ))} + +
LetterASCII Value
+ {row.node.letter} + + {row.node.value} +
+
+ ) + } +} + +export default IndexComponent + +export const IndexQuery = graphql` + query IndexQuery { + allLettersCsv { + edges { + node { + letter + value + } + } + } + } +`