Fully wired lang in.

src/main/scala/com/gravity/goose/Crawler.scala

@@ -58,6 +58,7 @@ class Crawler(config: Configuration) {
       parseCandidate <- URLHelper.getCleanedUrl(crawlCandidate.url)
       rawHtml <- getHTML(crawlCandidate, parseCandidate)
       doc <- getDocument(parseCandidate.url.toString, rawHtml)
+      lang = crawlCandidate.lang
     } {
       trace("Crawling url: " + parseCandidate.url)
 
@@ -82,7 +83,7 @@ class Crawler(config: Configuration) {
       // before we do any calcs on the body itself let's clean up the document
       article.doc = docCleaner.clean(article)
 
-      extractor.calculateBestNodeBasedOnClustering(article) match {
+      extractor.calculateBestNodeBasedOnClustering(article, lang) match {
         case Some(node: Element) => {
           article.topNode = node
           article.movies = extractor.extractVideos(article.topNode)
@@ -102,12 +103,10 @@ class Crawler(config: Configuration) {
               }
             }
           }
-          article.topNode = extractor.postExtractionCleanup(article.topNode)
+          article.topNode = extractor.postExtractionCleanup(article.topNode, lang)
 
 
-
-
-          article.cleanedArticleText = outputFormatter.getFormattedText(article.topNode)
+          article.cleanedArticleText = outputFormatter.getFormattedText(article.topNode, lang)
         }
         case _ => trace("NO ARTICLE FOUND")
       }

src/main/scala/com/gravity/goose/extractors/ContentExtractor.scala

@@ -191,7 +191,8 @@ trait ContentExtractor {
   * @return
   */
 
-  def calculateBestNodeBasedOnClustering(article: Article): Option[Element] = {
+  def calculateBestNodeBasedOnClustering(article: Article,
+                                         lang:String): Option[Element] = {
     trace(logPrefix + "Starting to calculate TopNode")
     val doc = article.doc
     var topNode: Element = null
@@ -203,7 +204,7 @@ trait ContentExtractor {
     val nodesWithText = mutable.Buffer[Element]()
     for (node <- nodesToCheck) {
       val nodeText: String = node.text
-      val wordStats: WordStats = StopWords.getStopWordCount(nodeText)
+      val wordStats: WordStats = StopWords.getStopWordCount(nodeText, lang)
       val highLinkDensity: Boolean = isHighLinkDensity(node)
       if (wordStats.getStopWordCount > 2 && !highLinkDensity) {
         nodesWithText.add(node)
@@ -217,7 +218,7 @@ trait ContentExtractor {
 
     for (node <- nodesWithText) {
       var boostScore: Float = 0
-      if (isOkToBoost(node)) {
+      if (isOkToBoost(node, lang)) {
         if (cnt >= 0) {
           boostScore = ((1.0 / startingBoost) * 50).asInstanceOf[Float]
           startingBoost += 1
@@ -237,7 +238,7 @@ trait ContentExtractor {
       trace(logPrefix + "Location Boost Score: " + boostScore + " on interation: " + i + "' id='" + node.parent.id + "' class='" + node.parent.attr("class"))
 
       val nodeText: String = node.text
-      val wordStats: WordStats = StopWords.getStopWordCount(nodeText)
+      val wordStats: WordStats = StopWords.getStopWordCount(nodeText, lang)
       val upscore: Int = (wordStats.getStopWordCount + boostScore).asInstanceOf[Int]
       updateScore(node.parent, upscore)
       updateScore(node.parent.parent, upscore / 2)
@@ -292,7 +293,7 @@ trait ContentExtractor {
   * @param node
   * @return
   */
-  private def isOkToBoost(node: Element): Boolean = {
+  private def isOkToBoost(node: Element, lang: String): Boolean = {
     val para = "p"
     var stepsAway: Int = 0
     val minimumStopWordCount = 5
@@ -306,7 +307,7 @@ trait ContentExtractor {
             return false
           }
           val paraText: String = currentNode.text
-          val wordStats: WordStats = StopWords.getStopWordCount(paraText)
+          val wordStats: WordStats = StopWords.getStopWordCount(paraText, lang)
           if (wordStats.getStopWordCount > minimumStopWordCount) {
             trace(logPrefix + "We're gonna boost this node, seems contenty")
             return true
@@ -491,10 +492,10 @@ trait ContentExtractor {
   * @param targetNode
   * @return
   */
-  def postExtractionCleanup(targetNode: Element): Element = {
+  def postExtractionCleanup(targetNode: Element, lang: String): Element = {
 
     trace(logPrefix + "Starting cleanup Node")
-    val node = addSiblings(targetNode)
+    val node = addSiblings(targetNode, lang)
     for {
       e <- node.children
       if (e.tagName != "p")
@@ -534,7 +535,9 @@ trait ContentExtractor {
   * @param currentSibling
   * @return
   */
-  def getSiblingContent(currentSibling: Element, baselineScoreForSiblingParagraphs: Int): Option[String] = {
+  def getSiblingContent(currentSibling: Element,
+                        baselineScoreForSiblingParagraphs: Int,
+                        lang: String): Option[String] = {
 
     if (currentSibling.tagName == "p" && currentSibling.text.length() > 0) {
       Some(currentSibling.outerHtml)
@@ -549,7 +552,7 @@ trait ContentExtractor {
         Some((for {
           firstParagraph <- potentialParagraphs
           if (firstParagraph.text.length() > 0)
-          wordStats: WordStats = StopWords.getStopWordCount(firstParagraph.text)
+          wordStats: WordStats = StopWords.getStopWordCount(firstParagraph.text, lang)
           paragraphScore: Int = wordStats.getStopWordCount
           siblingBaseLineScore: Double = .30
           if ((baselineScoreForSiblingParagraphs * siblingBaseLineScore).toDouble < paragraphScore)
@@ -578,14 +581,14 @@ trait ContentExtractor {
     b
   }
 
-  private def addSiblings(topNode: Element): Element = {
+  private def addSiblings(topNode: Element, lang: String): Element = {
 
     trace(logPrefix + "Starting to add siblings")
 
-    val baselineScoreForSiblingParagraphs: Int = getBaselineScoreForSiblings(topNode)
+    val baselineScoreForSiblingParagraphs: Int = getBaselineScoreForSiblings(topNode, lang)
     val results = walkSiblings(topNode) {
       currentNode => {
-        getSiblingContent(currentNode, baselineScoreForSiblingParagraphs)
+        getSiblingContent(currentNode, baselineScoreForSiblingParagraphs, lang)
 
       }
     }.reverse.flatMap(itm => itm)
@@ -602,15 +605,15 @@ trait ContentExtractor {
   * @param topNode
   * @return
   */
-  private def getBaselineScoreForSiblings(topNode: Element): Int = {
+  private def getBaselineScoreForSiblings(topNode: Element, lang: String): Int = {
     var base: Int = 100000
     var numberOfParagraphs: Int = 0
     var scoreOfParagraphs: Int = 0
     val nodesToCheck: Elements = topNode.getElementsByTag("p")
 
     for (node <- nodesToCheck) {
       val nodeText: String = node.text
-      val wordStats: WordStats = StopWords.getStopWordCount(nodeText)
+      val wordStats: WordStats = StopWords.getStopWordCount(nodeText, lang)
       val highLinkDensity: Boolean = isHighLinkDensity(node)
       if (wordStats.getStopWordCount > 2 && !highLinkDensity) {
         numberOfParagraphs += 1;

src/main/scala/com/gravity/goose/outputformatters/OutputFormatter.scala

@@ -49,11 +49,11 @@ trait OutputFormatter {
   * @param topNode the top most node to format
   * @return the prepared Element
   */
-  @Deprecated def getFormattedElement(topNode: Element): Element = {
+  @Deprecated def getFormattedElement(topNode: Element, lang: String): Element = {
     removeNodesWithNegativeScores(topNode)
     convertLinksToText(topNode)
     replaceTagsWithText(topNode)
-    removeParagraphsWithFewWords(topNode)
+    removeParagraphsWithFewWords(topNode, lang)
     topNode
   }
 
@@ -62,11 +62,11 @@ trait OutputFormatter {
   * @param topNode the top most node to format
   * @return a formatted string with all HTML removed
   */
-  def getFormattedText(topNode: Element): String = {
+  def getFormattedText(topNode: Element, lang: String): String = {
     removeNodesWithNegativeScores(topNode)
     convertLinksToText(topNode)
     replaceTagsWithText(topNode)
-    removeParagraphsWithFewWords(topNode)
+    removeParagraphsWithFewWords(topNode, lang)
     convertToText(topNode)
   }
 
@@ -173,7 +173,7 @@ trait OutputFormatter {
   /**
   * remove paragraphs that have less than x number of words, would indicate that it's some sort of link
   */
-  private def removeParagraphsWithFewWords(topNode: Element) {
+  private def removeParagraphsWithFewWords(topNode: Element, lang: String) {
     if (topNode != null) {
       if (logger.isDebugEnabled) {
         logger.debug("removeParagraphsWithFewWords starting...")
@@ -183,7 +183,7 @@ trait OutputFormatter {
 
       for (el <- allNodes) {
         try {
-          val stopWords = StopWords.getStopWordCount(el.text)
+          val stopWords = StopWords.getStopWordCount(el.text, lang)
           if (stopWords.getStopWordCount < 3 && el.getElementsByTag("object").size == 0 && el.getElementsByTag("embed").size == 0) {
             logger.debug("removeParagraphsWithFewWords - swcnt: %d removing text: %s".format(stopWords.getStopWordCount, el.text()))
             el.remove()