Gatsby copy linked files, video tag support, a tag support and option…

… to ignore default ignored file types (#1571)

* More features

Added

- ability to ignore specific file types and override defaults
- Ability to copy all image types if desired
- parsing of html a tags
- parsing of html video tags

* Format

* Fix link errors

* Update README

* Update gitignore

* Copy tweak

* Check the url is set

* Comment tweak

* Move plugin to dependencies to package will build on netlify

* Hopefully this fixes build

* Oooops, move back to devDependencies

packages/gatsby-remark-copy-linked-files/.gitignore

@@ -1 +1,2 @@
-/index.js
+/*.js
+yarn.lock

packages/gatsby-remark-copy-linked-files/README.md

@@ -8,14 +8,45 @@ Copies files linked to from markdown to your `public` folder.
 
 ## How to use
 
+#### Basic usage
+
 ```javascript
 // In your gatsby-config.js
 plugins: [
   {
     resolve: `gatsby-transformer-remark`,
     options: {
       plugins: [
-        `gatsby-remark-copy-linked-files`,
+        'gatsby-remark-copy-linked-files',
+      ]
+    }
+  }
+]
+```
+
+### How to override which file types are ignored
+
+```javascript
+// In your gatsby-config.js
+plugins: [
+  {
+    resolve: `gatsby-transformer-remark`,
+    options: {
+      plugins: [
+        { 
+          resolve: 'gatsby-remark-copy-linked-files',
+          options: {
+            // `ignoreFileExtensions` defaults to [`png`, `jpg`, `jpeg`, `bmp`, `tiff`]
+            // as we assume you'll use gatsby-remark-images to handle
+            // images in markdown as it automatically creates responsive
+            // versions of images.
+            //
+            // If you'd like to not use gatsby-remark-images and just copy your
+            // original images to the public directory, set
+            // `ignoreFileExtensions` to an empty array.
+            ignoreFileExtensions: [],
+          },
+        }
       ]
     }
   }
@@ -42,3 +73,14 @@ it.
 `my-awesome-pdf.pdf` should be in the same directory as the markdown
 file. When you build your site, the file will be copied to the `public`
 folder and the markdown HTML will be modified to point to it.
+
+### Supported Markdown tags
+
+- img
+- link
+
+### Supported HTML tags
+
+- <img />
+- <video />
+- <a />

packages/gatsby-remark-copy-linked-files/src/index.js

@@ -3,9 +3,19 @@ const isRelativeUrl = require(`is-relative-url`)
 const fsExtra = require(`fs-extra`)
 const path = require(`path`)
 const _ = require(`lodash`)
-const $ = require(`cheerio`)
+const cheerio = require(`cheerio`)
+const sizeOf = require(`image-size`)
+
+module.exports = (
+  { files, markdownNode, markdownAST, getNode },
+  pluginOptions
+) => {
+  const defaults = {
+    ignoreFileExtensions: [`png`, `jpg`, `jpeg`, `bmp`, `tiff`],
+  }
+
+  const options = _.defaults(pluginOptions, defaults)
 
-module.exports = ({ files, markdownNode, markdownAST, getNode }) => {
   const filesToCopy = new Map()
   // Copy linked files to the public directory and modify the AST to point to
   // new location of the files.
@@ -27,6 +37,12 @@ module.exports = ({ files, markdownNode, markdownAST, getNode }) => {
           `public`,
           `${linkNode.internal.contentDigest}.${linkNode.extension}`
         )
+
+        // Prevent uneeded copying
+        if (linkPath === newPath) {
+          return
+        }
+
         const relativePath = path.join(
           `/${linkNode.internal.contentDigest}.${linkNode.extension}`
         )
@@ -37,51 +53,181 @@ module.exports = ({ files, markdownNode, markdownAST, getNode }) => {
     }
   }
 
+  // Takes a node and generates the needed images and then returns
+  // the needed HTML replacement for the image
+  const generateImagesAndUpdateNode = async function(image) {
+    const imagePath = path.posix.join(
+      getNode(markdownNode.parent).dir,
+      image.attr(`src`)
+    )
+    const imageNode = _.find(files, file => {
+      if (file && file.absolutePath) {
+        return file.absolutePath === imagePath
+      }
+      return null
+    })
+    if (!imageNode || !imageNode.absolutePath) {
+      return
+    }
+
+    const initialImageSrc = image.attr(`src`)
+    // The link object will be modified to the new location so we'll
+    // use that data to update our ref
+    const link = { url: image.attr(`src`) }
+    await visitor(link)
+    image.attr(`src`, link.url)
+
+    let dimensions
+
+    if (!image.attr(`width`) || !image.attr(`height`)) {
+      dimensions = sizeOf(imageNode.absolutePath)
+    }
+
+    // Generate default alt tag
+    const srcSplit = initialImageSrc.split(`/`)
+    const fileName = srcSplit[srcSplit.length - 1]
+    const fileNameNoExt = fileName.replace(/\.[^/.]+$/, ``)
+    const defaultAlt = fileNameNoExt.replace(/[^A-Z0-9]/gi, ` `)
+
+    image.attr(`alt`, image.attr(`alt`) ? image.attr(`alt`) : defaultAlt)
+    image.attr(
+      `width`,
+      image.attr(`width`) ? image.attr(`width`) : dimensions.width
+    )
+    image.attr(
+      `height`,
+      image.attr(`height`) ? image.attr(`height`) : dimensions.height
+    )
+  }
+
   visit(markdownAST, `link`, link => {
+    const ext = link.url.split(`.`).pop()
+    if (options.ignoreFileExtensions.includes(ext)) {
+      return
+    }
+
     visitor(link)
   })
 
-  // Also copy gifs since Sharp can't process them as well as svgs since we
-  // exclude them from the image processing pipeline in
-  // gatsby-remark-images. This will only work for markdown img tags
+  // This will only work for markdown img tags
   visit(markdownAST, `image`, image => {
+    const ext = image.url.split(`.`).pop()
+    if (options.ignoreFileExtensions.includes(ext)) {
+      return
+    }
+
     const imagePath = path.join(getNode(markdownNode.parent).dir, image.url)
     const imageNode = _.find(files, file => {
       if (file && file.absolutePath) {
         return file.absolutePath === imagePath
       }
       return false
     })
-    if (
-      imageNode &&
-      (imageNode.extension === `gif` || imageNode.extension === `svg`)
-    ) {
+
+    if (imageNode) {
       visitor(image)
     }
   })
 
-  // Same as the above except it only works for html img tags
-  visit(markdownAST, `html`, node => {
-    if (node.value.startsWith(`<img`)) {
-      let image = Object.assign(node, $.parseHTML(node.value)[0].attribs)
-      image.url = image.src
-      image.type = `image`
-      image.position = node.position
+  // For each HTML Node
+  visit(markdownAST, `html`, async node => {
+    const $ = cheerio.load(node.value)
+    // Handle Images
+    const imageRefs = []
+    $(`img`).each(function() {
+      try {
+        if (isRelativeUrl($(this).attr(`src`))) {
+          imageRefs.push($(this))
+        }
+      } catch (err) {
+        // Ignore
+      }
+    })
 
-      const imagePath = path.join(getNode(markdownNode.parent).dir, image.url)
-      const imageNode = _.find(files, file => {
-        if (file && file.absolutePath) {
-          return file.absolutePath === imagePath
+    for (let thisImg of imageRefs) {
+      try {
+        const ext = thisImg
+          .attr(`src`)
+          .split(`.`)
+          .pop()
+        if (options.ignoreFileExtensions.includes(ext)) {
+          return
         }
-        return false
-      })
-      if (
-        imageNode &&
-        (imageNode.extension === `gif` || imageNode.extension === `svg`)
-      ) {
-        visitor(image)
+
+        await generateImagesAndUpdateNode(thisImg)
+      } catch (err) {
+        // Ignore
+      }
+    }
+
+    const videoRefs = []
+    // Handle video tags.
+    $(`video source`).each(function() {
+      try {
+        if (isRelativeUrl($(this).attr(`src`))) {
+          videoRefs.push($(this))
+        }
+      } catch (err) {
+        // Ignore
+      }
+    })
+
+    for (let thisVideo of videoRefs) {
+      try {
+        const ext = thisVideo
+          .attr(`src`)
+          .split(`.`)
+          .pop()
+        if (options.ignoreFileExtensions.includes(ext)) {
+          return
+        }
+
+        // The link object will be modified to the new location so we'll
+        // use that data to update our ref
+        const link = { url: thisVideo.attr(`src`) }
+        await visitor(link)
+        thisVideo.attr(`src`, link.url)
+      } catch (err) {
+        // Ignore
       }
     }
+
+    // Handle a tags.
+    const aRefs = []
+    $(`a`).each(function() {
+      try {
+        if (isRelativeUrl($(this).attr(`href`))) {
+          aRefs.push($(this))
+        }
+      } catch (err) {
+        // Ignore
+      }
+    })
+
+    for (let thisATag of aRefs) {
+      try {
+        const ext = thisATag
+          .attr(`href`)
+          .split(`.`)
+          .pop()
+        if (options.ignoreFileExtensions.includes(ext)) {
+          return
+        }
+
+        // The link object will be modified to the new location so we'll
+        // use that data to update our ref
+        const link = { url: thisATag.attr(`href`) }
+        await visitor(link)
+        thisATag.attr(`href`, link.url)
+      } catch (err) {
+        // Ignore
+      }
+    }
+
+    // Replace the image node with an inline HTML node.
+    node.type = `html`
+    node.value = $.html()
+    return
   })
 
   return Promise.all(

packages/gatsby-remark-images/src/index.js

@@ -61,7 +61,6 @@ module.exports = (
       args: options,
     })
 
-    // console.log("responsiveSizesResult", responsiveSizesResult)
     // Calculate the paddingBottom %
     const ratio = `${1 / responsiveSizesResult.aspectRatio * 100}%`
 
@@ -169,12 +168,16 @@ module.exports = (
             })
 
             for (let thisImg of imageRefs) {
-              //Get the details we need
+              // Get the details we need.
               let formattedImgTag = {}
               formattedImgTag.url = thisImg.attr(`src`)
               formattedImgTag.title = thisImg.attr(`title`)
               formattedImgTag.alt = thisImg.attr(`alt`)
 
+              if (!formattedImgTag.url) {
+                return resolve()
+              }
+
               const fileType = formattedImgTag.url.slice(-3)
 
               // Ignore gifs as we can't process them,

scripts/publish-site.sh

@@ -1,6 +1,6 @@
 echo "=== Building ES5 version of Gatsby"
 rm -r node_modules yarn.lock
-NODE_ENV=development yarn
+NODE_ENV=development yarn bootstrap
 ./node_modules/.bin/lerna run build
 
 yarn global add gatsby-dev-cli