Skip to content

Commit

Permalink
trunk: Updating svn:ignore properties. These are now extracted from t…
Browse files Browse the repository at this point in the history
…he .gitignore file using the script misc/maintenance/svnignore.sh.

    Adding 'unlink' commands to various tests that write to files (this process is not complete).
    Some changes to the online-decoding setup as it relates to pitch (this work is not yet finished, but checking it in with other changes,
      as it shouldn't break anything that currently runs.)



git-svn-id: https://svn.code.sf.net/p/kaldi/code/trunk@4365 5e6a8d80-dfce-4ca6-a32a-6e07a63d50c8
  • Loading branch information
danpovey committed Sep 1, 2014
1 parent d0ef62c commit f62e51e
Show file tree
Hide file tree
Showing 19 changed files with 506 additions and 58 deletions.
240 changes: 225 additions & 15 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -12,24 +12,37 @@
*.la
*.a

# some weird thing that MacOs creates.
*.dSYM

# /tools/
tools/ATLAS/
tools/atlas3.8.3.tar.gz
tools/irstlm/
tools/openfst
tools/openfst-1.3.2.tar.gz
tools/openfst-1.3.2/
tools/openfst-1.3.4.tar.gz
tools/openfst-1.3.4/
tools/pa_stable_v19_20111121.tgz
tools/portaudio/
tools/sctk-2.4.0-20091110-0958.tar.bz2
tools/sctk-2.4.0/
tools/sph2pipe_v2.5.tar.gz
tools/sph2pipe_v2.5/
/tools/ATLAS/
/tools/atlas3.8.3.tar.gz
/tools/irstlm/
/tools/openfst
/tools/openfst-1.3.2.tar.gz
/tools/openfst-1.3.2/
/tools/openfst-1.3.4.tar.gz
/tools/openfst-1.3.4/
/tools/pa_stable_v19_20111121.tgz
/tools/portaudio/
/tools/sctk-2.4.0-20091110-0958.tar.bz2
/tools/sctk-2.4.0/
/tools/sph2pipe_v2.5.tar.gz
/tools/sph2pipe_v2.5/
/tools/kaldi_lm.tar.gz
/tools/sctk-2.4.8-20130429-2145.tar.bz2
/tools/OpenBLAS
/tools/sctk
/tools/sctk-2.4.8
/tools/srilm
/tools/CLAPACK_include
/tools/kaldi_lm

# /src/
/src/kaldi.mk*
/src/kaldi.mk
/src/kaldi.mk.tmp
/src/kaldi.mk.bak

# /src/base/
/src/base/.depend.mk
Expand Down Expand Up @@ -627,3 +640,200 @@ tools/sph2pipe_v2.5/
/src/util/kaldi-io-test
/src/util/text-utils-test
/src/util/parse-options-test


# we should probably have more things like this in
# the .gitignore:
/egs/callhome_egyptian/s5/mfcc
/egs/callhome_egyptian/s5/data
/egs/callhome_egyptian/s5/exp

/egs/ami/s5/mfcc
/egs/ami/s5/plp
/egs/ami/s5/exp
/egs/ami/s5/data
/egs/aurora4/s5/mfcc
/egs/aurora4/s5/plp
/egs/aurora4/s5/exp
/egs/aurora4/s5/data
/egs/babel/s5/mfcc
/egs/babel/s5/plp
/egs/babel/s5/exp
/egs/babel/s5/data
/egs/babel/s5b/mfcc
/egs/babel/s5b/plp
/egs/babel/s5b/exp
/egs/babel/s5b/data
/egs/callhome_egyptian/s5/mfcc
/egs/callhome_egyptian/s5/plp
/egs/callhome_egyptian/s5/exp
/egs/callhome_egyptian/s5/data
/egs/chime_wsj0/s5/mfcc
/egs/chime_wsj0/s5/plp
/egs/chime_wsj0/s5/exp
/egs/chime_wsj0/s5/data
/egs/fisher_english/s5/mfcc
/egs/fisher_english/s5/plp
/egs/fisher_english/s5/exp
/egs/fisher_english/s5/data
/egs/fisher_swbd/s5/mfcc
/egs/fisher_swbd/s5/plp
/egs/fisher_swbd/s5/exp
/egs/fisher_swbd/s5/data
/egs/gale_arabic/s5/mfcc
/egs/gale_arabic/s5/plp
/egs/gale_arabic/s5/exp
/egs/gale_arabic/s5/data
/egs/gp/s1/mfcc
/egs/gp/s1/plp
/egs/gp/s1/exp
/egs/gp/s1/data
/egs/gp/s5/mfcc
/egs/gp/s5/plp
/egs/gp/s5/exp
/egs/gp/s5/data
/egs/hkust/s5/mfcc
/egs/hkust/s5/plp
/egs/hkust/s5/exp
/egs/hkust/s5/data
/egs/rm/s4/mfcc
/egs/rm/s4/plp
/egs/rm/s4/exp
/egs/rm/s4/data
/egs/rm/s5/mfcc
/egs/rm/s5/plp
/egs/rm/s5/exp
/egs/rm/s5/data
/egs/rm/s6/mfcc
/egs/rm/s6/plp
/egs/rm/s6/exp
/egs/rm/s6/data
/egs/sprakbanken/s5/mfcc
/egs/sprakbanken/s5/plp
/egs/sprakbanken/s5/exp
/egs/sprakbanken/s5/data
/egs/swbd/s5/mfcc
/egs/swbd/s5/plp
/egs/swbd/s5/exp
/egs/swbd/s5/data
/egs/swbd/s5b/mfcc
/egs/swbd/s5b/plp
/egs/swbd/s5b/exp
/egs/swbd/s5b/data
/egs/tedlium/s5/mfcc
/egs/tedlium/s5/plp
/egs/tedlium/s5/exp
/egs/tedlium/s5/data
/egs/tidigits/s5/mfcc
/egs/tidigits/s5/plp
/egs/tidigits/s5/exp
/egs/tidigits/s5/data
/egs/timit/s3/mfcc
/egs/timit/s3/plp
/egs/timit/s3/exp
/egs/timit/s3/data
/egs/timit/s4/mfcc
/egs/timit/s4/plp
/egs/timit/s4/exp
/egs/timit/s4/data
/egs/timit/s5/mfcc
/egs/timit/s5/plp
/egs/timit/s5/exp
/egs/timit/s5/data
/egs/voxforge/s5/mfcc
/egs/voxforge/s5/plp
/egs/voxforge/s5/exp
/egs/voxforge/s5/data
/egs/vystadial_cz/s5/mfcc
/egs/vystadial_cz/s5/plp
/egs/vystadial_cz/s5/exp
/egs/vystadial_cz/s5/data
/egs/vystadial_en/s5/mfcc
/egs/vystadial_en/s5/plp
/egs/vystadial_en/s5/exp
/egs/vystadial_en/s5/data
/egs/wsj/s5/mfcc
/egs/wsj/s5/plp
/egs/wsj/s5/exp
/egs/wsj/s5/data
/egs/yesno/s5/mfcc
/egs/yesno/s5/plp
/egs/yesno/s5/exp
/egs/yesno/s5/data
/src/bin/est-pca
/src/bin/matrix-sum-rows
/src/bin/sum-lda-accs
/src/bin/sum-mllt-accs
/src/bin/transform-vec
/src/cudamatrix/cu-array-test
/src/cudamatrix/cu-block-matrix-test
/src/cudamatrix/cu-math-test
/src/cudamatrix/cu-matrix-speed-test
/src/cudamatrix/cu-matrix-test
/src/cudamatrix/cu-packed-matrix-test
/src/cudamatrix/cu-sp-matrix-speed-test
/src/cudamatrix/cu-sp-matrix-test
/src/cudamatrix/cu-test
/src/cudamatrix/cu-tp-matrix-test
/src/cudamatrix/cu-vector-speed-test
/src/cudamatrix/cu-vector-test
/src/feat/feature-functions-test
/src/feat/feature-sdc-test
/src/feat/pitch-functions-test
/src/featbin/append-vector-to-feats
/src/featbin/compute-and-process-kaldi-pitch-feats
/src/featbin/modify-cmvn-stats
/src/featbin/wav-copy
/src/fstext/push-special-test
/src/gmmbin/gmm-acc-mllt-global
/src/gmmbin/gmm-est-fmllr-global
/src/gmmbin/gmm-global-est-lvtln-trans
/src/gmmbin/gmm-global-get-post
/src/gmmbin/gmm-global-gselect-to-post
/src/gmmbin/gmm-latgen-faster-regtree-fmllr
/src/gmmbin/gmm-transform-means-global
/src/ivector/ivector-extractor-test
/src/ivector/logistic-regression-test
/src/ivector/plda-test
/src/ivectorbin/create-split-from-vad
/src/ivectorbin/ivector-adapt-plda
/src/ivectorbin/ivector-extract-online
/src/ivectorbin/logistic-regression-copy
/src/ivectorbin/logistic-regression-eval
/src/ivectorbin/logistic-regression-train
/src/lat/determinize-lattice-pruned-test
/src/lat/minimize-lattice-test
/src/lat/push-lattice-test
/src/latbin/lattice-confidence
/src/latbin/lattice-expand-ngram
/src/nnet/nnet-component-test
/src/nnet/nnet-randomizer-test
/src/nnet2/nnet-example-functions-test
/src/nnet2/nnet-precondition-online-test
/src/nnet2/nnet-precondition-test
/src/nnet2bin/cuda-compiled
/src/nnet2bin/nnet-adjust-priors
/src/nnet2bin/nnet-am-switch-preconditioning
/src/nnet2bin/nnet-replace-last-layers
/src/nnet2bin/nnet-to-raw-nnet
/src/nnet2bin/nnet-train-parallel-perturbed
/src/nnet2bin/nnet-train-simple-perturbed
/src/nnet2bin/nnet1-to-raw-nnet
/src/nnet2bin/raw-nnet-copy
/src/online2bin/apply-cmvn-online
/src/online2bin/compress-uncompress-speex
/src/online2bin/extend-wav-with-silence
/src/online2bin/ivector-extract-online2
/src/online2bin/ivector-randomize
/src/online2bin/online2-wav-dump-features
/src/online2bin/online2-wav-gmm-latgen-faster
/src/online2bin/online2-wav-nnet2-latgen-faster
/src/sgmm/estimate-am-sgmm-multi-test
/src/sgmm2/am-sgmm2-test
/src/sgmm2/estimate-am-sgmm2-test
/src/sgmm2/fmllr-sgmm2-test
/src/thread/kaldi-task-sequence-test
/src/thread/kaldi-thread-test
/src/transform/fmllr-raw-test
/src/util/simple-options-test

45 changes: 45 additions & 0 deletions egs/wsj/s5/conf/online_pitch.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
## This config is given by conf/make_pitch_online.sh to the program compute-and-process-kaldi-pitch-feats,
## and is copied by steps/online/nnet2/prepare_online_decoding.sh and similar scripts, to be given
## to programs like online2-wav-nnet2-latgen-faster.
## The program compute-and-process-kaldi-pitch-feats will use it to compute pitch features that
## are the same as that those which will generated in online decoding; this enables us to train
## in a way that's compatible with online decoding.
##

## most of these options relate to the post-processing rather than the pitch
## extraction itself.
--add-raw-log-pitch=true ## this is intended for input to neural nets, so our
## approach is "throw everything in and see what
## sticks".
--normalization-left-context=100
--normalization-right-context=10 # We're removing amost all the right-context
# for the normalization. The reason why we
# include a small nonzero right-context (of
# just 0.1 second) is that by adding a little
# latency to the computation, it enables us to
# get a more accurate estimate of the pitch on
# the frame we're currently computing the
# normalized pitch of. We know for the current
# frame that we will have at least 10 frames to
# the right, and those extra 10 frames will
# increase the quality of the Viterbi
# backtrace.
#
# Note: our changes to the (left,right) context
# from the defaults of (75,75) to (100,10) will
# almost certainly worsen results, but will
# reduce latency.
--frames-per-chunk=10 ## relates to offline simulation of online decoding; 1
## would be equivalent to getting in samples one by
## one.
--simulate-first-pass-online=true ## this make the online-pitch-extraction code
## output the 'first-pass' features, which
## are less accurate than the final ones, and
## which are the only features the neural-net
## decoding would ever see (since we can't
## afford to do lattice rescoring in the
## neural-net code
--delay=5 ## We delay all the pitch information by 5 frames. This is almost
## certainly not helpful, but it helps to reduce the overall latency
## added by the pitch computation, from 10 (given by
## --normalization-right-context) to 10 - 5 = 5.
19 changes: 12 additions & 7 deletions egs/wsj/s5/steps/online/nnet2/prepare_online_decoding.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,10 @@ add_pitch=false
mfcc_config=conf/mfcc.conf # you can override any of these you need to override.
plp_config=conf/plp.conf
fbank_config=conf/fbank.conf
pitch_config=conf/pitch.conf
pitch_process_config=conf/pitch_process.conf
# online_pitch_config is the config file for both pitch extraction and
# post-processing; we combine them into one because during training this
# is given to the program compute-and-process-kaldi-pitch-feats.
online_pitch_config=conf/online_pitch.conf

# Below are some options that affect the iVectors, and should probably
# match those used in extract_ivectors_online.sh.
Expand Down Expand Up @@ -132,12 +134,15 @@ fi
if $add_pitch; then
echo "$0: enabling pitch features (note: this has not been tested)"
echo "--add-pitch=true" >>$conf
echo "$0: creating $dir/conf/pitch.conf"
echo "--pitch-config=$dir/conf/pitch.conf" >>$conf
cp $pitch_config $dir/conf/pitch.conf || exit 1;
echo "--pitch-process-config=$dir/conf/pitch_process.conf" >>$conf
cp $pitch_process_config $dir/conf/pitch_process.conf || exit 1;
echo "$0: creating $dir/conf/online_pitch.conf"
if [ ! -f $online_pitch_config ]; then
echo "$0: expected file '$online_pitch_config' to exist.";
exit 1;
fi
cp $online_pitch_config $dir/conf/online_pitch.conf || exit 1;
echo "--online-pitch-config=$dir/conf/online_pitch.conf" >>$conf
fi

silphonelist=`cat $lang/phones/silence.csl` || exit 1;
echo "--endpoint.silence-phones=$silphonelist" >>$conf
echo "$0: created config file $conf"
Expand Down
5 changes: 3 additions & 2 deletions egs/wsj/s5/steps/paste_feats.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
# Copyright 2014 Brno University of Technology (Author: Karel Vesely)
# Copyright 2012 Johns Hopkins University (Author: Daniel Povey)
# Apache 2.0
# This script appends the features in two data directories.
# This script appends the features in two or more data directories.

# To be run from .. (one directory up from here)
# see ../run.sh for example
Expand All @@ -22,6 +22,7 @@ if [ -f path.sh ]; then . ./path.sh; fi

if [ $# -lt 5 ]; then
echo "usage: $0 [options] <src-data-dir1> <src-data-dir2> [<src-data-dirN>] <dest-data-dir> <log-dir> <path-to-storage-dir>";
echo "e.g.: $0 data/train_mfcc data/train_bottleneck data/train_combined exp/append_mfcc_plp mfcc"
echo "options: "
echo " --cmd (utils/run.pl|utils/queue.pl <queue opts>) # how to run jobs."
exit 1;
Expand Down Expand Up @@ -71,7 +72,7 @@ done > $data/feats.scp || exit 1;
nf=`cat $data/feats.scp | wc -l`
nu=`cat $data/utt2spk | wc -l`
if [ $nf -ne $nu ]; then
echo "It seems not all of the feature files were successfully ($nf != $nu);"
echo "It seems not all of the feature files were successfully processed ($nf != $nu);"
echo "consider using utils/fix_data_dir.sh $data"
fi

Expand Down
Loading

0 comments on commit f62e51e

Please sign in to comment.