docs/api.html

<!DOCTYPE html>

<html xmlns="http://www.w3.org/1999/xhtml">

<head>

<meta charset="utf-8" />
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="generator" content="pandoc" />


<meta name="author" content="Dmitriy Selivanov" />


<title>text2vec</title>

<script src="site_libs/jquery-1.11.3/jquery.min.js"></script>
<meta name="viewport" content="width=device-width, initial-scale=1" />
<link href="site_libs/bootstrap-3.3.5/css/cosmo.min.css" rel="stylesheet" />
<script src="site_libs/bootstrap-3.3.5/js/bootstrap.min.js"></script>
<script src="site_libs/bootstrap-3.3.5/shim/html5shiv.min.js"></script>
<script src="site_libs/bootstrap-3.3.5/shim/respond.min.js"></script>
<script src="site_libs/jqueryui-1.11.4/jquery-ui.min.js"></script>
<link href="site_libs/tocify-1.9.1/jquery.tocify.css" rel="stylesheet" />
<script src="site_libs/tocify-1.9.1/jquery.tocify.js"></script>
<script src="site_libs/navigation-1.1/tabsets.js"></script>
<link href="site_libs/font-awesome-4.5.0/css/font-awesome.min.css" rel="stylesheet" />


<style type="text/css">
h1 {
  font-size: 34px;
}
h1.title {
  font-size: 38px;
}
h2 {
  font-size: 30px;
}
h3 {
  font-size: 24px;
}
h4 {
  font-size: 18px;
}
h5 {
  font-size: 16px;
}
h6 {
  font-size: 12px;
}
.table th:not([align]) {
  text-align: left;
}
</style>


</head>

<body>

<style type = "text/css">
.main-container {
  max-width: 940px;
  margin-left: auto;
  margin-right: auto;
}
code {
  color: inherit;
  background-color: rgba(0, 0, 0, 0.04);
}
img {
  max-width:100%;
  height: auto;
}
.tabbed-pane {
  padding-top: 12px;
}
button.code-folding-btn:focus {
  outline: none;
}
</style>


<style type="text/css">
/* padding for bootstrap navbar */
body {
  padding-top: 51px;
  padding-bottom: 40px;
}
/* offset scroll position for anchor links (for fixed navbar)  */
.section h1 {
  padding-top: 56px;
  margin-top: -56px;
}

.section h2 {
  padding-top: 56px;
  margin-top: -56px;
}
.section h3 {
  padding-top: 56px;
  margin-top: -56px;
}
.section h4 {
  padding-top: 56px;
  margin-top: -56px;
}
.section h5 {
  padding-top: 56px;
  margin-top: -56px;
}
.section h6 {
  padding-top: 56px;
  margin-top: -56px;
}
</style>

<script>
// manage active state of menu based on current page
$(document).ready(function () {
  // active menu anchor
  href = window.location.pathname
  href = href.substr(href.lastIndexOf('/') + 1)
  if (href === "")
    href = "index.html";
  var menuAnchor = $('a[href="' + href + '"]');

  // mark it active
  menuAnchor.parent().addClass('active');

  // if it's got a parent navbar menu mark it active as well
  menuAnchor.closest('li.dropdown').addClass('active');
});
</script>


<div class="container-fluid main-container">

<!-- tabsets -->
<script>
$(document).ready(function () {
  window.buildTabsets("TOC");
});
</script>

<!-- code folding -->


<script>
$(document).ready(function ()  {

    // move toc-ignore selectors from section div to header
    $('div.section.toc-ignore')
        .removeClass('toc-ignore')
        .children('h1,h2,h3,h4,h5').addClass('toc-ignore');

    // establish options
    var options = {
      selectors: "h1,h2,h3",
      theme: "bootstrap3",
      context: '.toc-content',
      hashGenerator: function (text) {
        return text.replace(/[.\\/?&!#<>]/g, '').replace(/\s/g, '_').toLowerCase();
      },
      ignoreSelector: ".toc-ignore",
      scrollTo: 0
    };
    options.showAndHide = false;
    options.smoothScroll = true;

    // tocify
    var toc = $("#TOC").tocify(options).data("toc-tocify");
});
</script>

<style type="text/css">

#TOC {
  margin: 25px 0px 20px 0px;
}
@media (max-width: 768px) {
#TOC {
  position: relative;
  width: 100%;
}
}


.toc-content {
  padding-left: 30px;
  padding-right: 40px;
}

div.main-container {
  max-width: 1200px;
}

div.tocify {
  width: 20%;
  max-width: 260px;
  max-height: 85%;
}

@media (min-width: 768px) and (max-width: 991px) {
  div.tocify {
    width: 25%;
  }
}

@media (max-width: 767px) {
  div.tocify {
    width: 100%;
    max-width: none;
  }
}

.tocify ul, .tocify li {
  line-height: 20px;
}

.tocify-subheader .tocify-item {
  font-size: 0.90em;
  padding-left: 25px;
  text-indent: 0;
}

.tocify .list-group-item {
  border-radius: 0px;
}

.tocify-subheader {
  display: inline;
}
.tocify-subheader .tocify-item {
  font-size: 0.95em;
}

</style>

<!-- setup 3col/9col grid for toc_float and main content  -->
<div class="row-fluid">
<div class="col-xs-12 col-sm-4 col-md-3">
<div id="TOC" class="tocify">
</div>
</div>

<div class="toc-content col-xs-12 col-sm-8 col-md-9">


<div class="navbar navbar-default  navbar-fixed-top" role="navigation">
  <div class="container">
    <div class="navbar-header">
      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar">
        <span class="icon-bar"></span>
        <span class="icon-bar"></span>
        <span class="icon-bar"></span>
      </button>
      <a class="navbar-brand" href="index.html">text2vec</a>
    </div>
    <div id="navbar" class="navbar-collapse collapse">
      <ul class="nav navbar-nav">
        <li>
  <a href="index.html">Home</a>
</li>
<li>
  <a href="vectorization.html">Vectorization</a>
</li>
<li>
  <a href="glove.html">GloVe</a>
</li>
<li>
  <a href="collocations.html">Collocations</a>
</li>
<li>
  <a href="topic_modeling.html">Topic modeling</a>
</li>
<li>
  <a href="similarity.html">Similarity</a>
</li>
<li>
  <a href="api.html">API</a>
</li>
      </ul>
      <ul class="nav navbar-nav navbar-right">
        <li class="dropdown">
  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
    <span class="fa fa-language"></span>
     
     
    <span class="caret"></span>
  </a>
  <ul class="dropdown-menu" role="menu">
    <li>
      <a href="http://text2vec.org/">English</a>
    </li>
    <li>
      <a href="https://cndocr.github.io/text2vec-doc-cn/">Chinese</a>
    </li>
  </ul>
</li>
<li>
  <a href="https://github.com/dselivanov/text2vec">
    <span class="fa fa-github"></span>
     
  </a>
</li>
<li>
  <a href="http://stackoverflow.com/questions/tagged/text2vec">
    <span class="fa fa-stack-overflow"></span>
     
  </a>
</li>
      </ul>
    </div><!--/.nav-collapse -->
  </div><!--/.container -->
</div><!--/.navbar -->

<div class="fluid-row" id="header">


<h1 class="title toc-ignore">text2vec</h1>
<h4 class="author"><em>Dmitriy Selivanov</em></h4>

</div>


<div id="api" class="section level1">
<h1>API</h1>
<p>Goals which we aimed to achieve as a result of development of <code>text2vec</code>:</p>
<ul>
<li><strong>Concise</strong> - expose as few functions as possible;</li>
<li><strong>Consistent</strong> - expose unified interfaces, no need to explore new interface for each task;</li>
<li><strong>Flexible</strong> - allow to easily solve complex tasks;</li>
<li><strong>Fast</strong> - maximize efficiency per single thread, transparently scale to multiple threads on multicore machines;</li>
<li><strong>Memory efficient</strong> - use streams and iterators, not keep data in RAM if possible.</li>
</ul>
<p>Conceptually we can divide API into several pieces:</p>
<div id="vectorization" class="section level2">
<h2>Vectorization</h2>
<p>See <a href="vectorization.html">Vectorization</a> section for details.</p>
<p><code>create_*</code> family functions, <code>vocab_vectorizer()</code> and <code>hash_vectorizer()</code> are made to create vocabularies, Document-Term matrices and Term co-occurence matrices. Simply this family of functions is in charge of converting text into numeric form. Main functions are:</p>
<ul>
<li><code>create_vocabulary()</code>;</li>
<li><code>create_dtm()</code>;</li>
<li><code>create_tcm()</code>;</li>
<li><code>vocab_vectorizer()</code>, <code>hash_vectorizer()</code>.</li>
</ul>
</div>
<div id="io-handling" class="section level2">
<h2>I/O handling</h2>
<p>All functions from <code>create_*</code> family work with <strong>iterators</strong> over tokens as input. Good examples for creation of such iterators are:</p>
<ul>
<li><code>ifiles()</code> for creation iterator over files. Note that text2vec doesn’t handle I/O, users should provide their own reader function (<code>data.table::fread()</code> and functions from <code>readr</code> package usually are good choices).</li>
<li><code>itoken()</code> for creation iterator over tokens;</li>
</ul>
<p>Once user needs some custom source (for example data stream from some RDBMS), he/she just needs to create correct iterator over tokens.</p>
<div id="easy-parallel-processing" class="section level3">
<h3>Easy parallel processing</h3>
<p><code>text2vec</code> also provides convenient functions for <strong>easy parallel processing</strong> of text (many of tasks are emrassingly parallel).</p>
<ul>
<li><code>ifiles_parallel()</code> same as <code>ifiles()</code> above, but creates <strong>parallel iterator</strong> if parallel backend is registered (for example with <code>registerDoParallel</code>)</li>
<li><code>itoken_parallel()</code> is the same as <code>itoken()</code> above but also creates <strong>parallel iterator</strong> if parallel backend is registered.</li>
</ul>
<p>Parallel <code>itoken</code> iterators can be used in <code>create_dtm()</code>, <code>create_tcm()</code> functions exatly the same way as sequential counterparts.</p>
</div>
</div>
<div id="models" class="section level2">
<h2>Models</h2>
<p>text2vec provides unified interface for models, which is inspired by <code>scikit-learn</code> interface. Models in text2vec are mostly <em>transformers</em> and <em>decompositions</em> - they transform Document-Term matrix or decompose into 2 low-rank matrices.</p>
<p>Models include:</p>
<ul>
<li>Tf-idf reweighting. See <a href="vectorization.html#tf-idf">Tf-idf in vectorization</a> section;</li>
<li>Global Vectors (<strong>GloVe</strong>) word embeddings. See <a href="glove.html">Word Embeddings</a> section;</li>
<li>Latent Semantic Analysis (<strong>LSA</strong>). See <a href="topic_modeling.html#latent_semantic_analysis">LSA</a> section;</li>
<li>Latent Dirichlet Allocation (<strong>LDA</strong>). See <a href="topic_modeling.html#latent_dirichlet_allocation">LDA</a> section.</li>
<li><strong>Collocations</strong>. Collocations model which can learn phrases from text is a bit separate from others and has a little bit different interface. It takes <code>itoken</code> iterator as input to <code>fit</code> method and learn model. After that user can pass another <code>itoken</code> iterator to <code>transform</code> method and receive another <code>itoken</code> iterator wich will produce tokens with phrases concatenated into single token.</li>
</ul>
<p><strong>All text2vec models are mutable! This means that <code>fit()</code> and <code>fit_transform()</code> methods change model which was provided as argument.</strong></p>
<div id="important-verbs" class="section level3">
<h3>Important verbs</h3>
<p>All models have unified interface. User should only remember few verbs for models manipulation:</p>
<ul>
<li><code>model$new(...)</code> - create model object, set up initial parameters for model. This is model-specific. For example for LDA it can be number of topics <span class="math inline">\(K\)</span>, alpha(<span class="math inline">\(\alpha\)</span>) and eta(<span class="math inline">\(\eta\)</span>) priors;</li>
<li><code>model$partial_fit(x, ...)</code> - partially fits model to data (for online models);</li>
<li><code>model$fit_transform(x, ...)</code> - fits model to data and then transforms data with fitted model;</li>
<li><code>model$transform(x_new, ...)</code> - transforms new data with pretrained model.</li>
</ul>
<p>Decomposition models decompose matrix into 2 low rank matrices <span class="math inline">\(X\)</span> and <span class="math inline">\(Y\)</span>. <span class="math inline">\(X\)</span> corresponds to item embeddings and <span class="math inline">\(Y\)</span> corresponds to feature embeddings. For example for <code>LDA</code> <span class="math inline">\(X\)</span> will be document-topic assignements and <span class="math inline">\(Y\)</span> will be topic-word assignements. While <code>fit_transform</code> or <code>transform</code> methods gives you <span class="math inline">\(X\)</span>, second, matrix <span class="math inline">\(Y\)</span> is available as <code>components</code> read-only field: <code>model$components</code>. Examples of “decomposition” models in <code>text2vec</code> are <code>LDA</code>, <code>LSA</code>, <code>GloVe</code>. Check documentation of these classes for additional information.</p>
</div>
</div>
<div id="distances" class="section level2">
<h2>Distances</h2>
<p>See <a href="distances.html">Distances</a> section for details.</p>
<p>text2vec package provides 2 set of functions for measuring various distances/similarity in a unified way. All methods are written with special attention to computational performance and memory efficiency.</p>
<ol style="list-style-type: decimal">
<li><code>sim2(x, y, method)</code> - calculates similarity between <strong>each row</strong> of matrix <code>x</code> and <strong>each row</strong> of matrix <code>y</code> using given <code>method</code>.</li>
<li><code>psim2(x, y, method)</code> - calculates <strong>p</strong>arallel similarity between rows of matrix <code>x</code> and <strong>corresponding</strong> rows of matrix <code>y</code> using given <code>method.</code></li>
<li><code>dist2(x, y, method)</code> - calculates distance/dissimilarity between <strong>each row</strong> of matrix <code>x</code> and <strong>each row</strong> of matrix <code>y</code> using given <code>method</code>.</li>
<li><code>pdist2(x, y, method)</code> - calculates <strong>p</strong>arallel distance/dissimilarity between rows of matrix <code>x</code> and <strong>corresponding</strong> rows of matrix <code>y</code> using given <code>method.</code></li>
</ol>
<p>Distances/similarities implemented at the moment:</p>
<ul>
<li>Cosine</li>
<li>Jaccard</li>
<li>Euclidean</li>
<li>Relaxed Word Mover’s Distance</li>
</ul>
</div>
</div>


<footer class="footer">
  <div class="text-muted"><strong>text2vec</strong> is created by <a href="http://www.dsnotes.com">Dmitry Selivanov</a> and contributors. &copy;  2016.</div>
  <div class="text-muted"> If you have found any BUGS please report them <a href="https://github.com/dselivanov/text2vec/issues">here</a>.</div>
</footer>

<script>
  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
  m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
  })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');


  ga('create', 'UA-56994099-2', 'auto');
  ga('send', 'pageview');


</script>


</div>
</div>

</div>

<script>

// add bootstrap table styles to pandoc tables
function bootstrapStylePandocTables() {
  $('tr.header').parent('thead').parent('table').addClass('table table-condensed');
}
$(document).ready(function () {
  bootstrapStylePandocTables();
});


</script>

<!-- dynamically load mathjax for compatibility with self-contained -->
<script>
  (function () {
    var script = document.createElement("script");
    script.type = "text/javascript";
    script.src  = "https://mathjax.rstudio.com/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML";
    document.getElementsByTagName("head")[0].appendChild(script);
  })();
</script>

</body>
</html>