Add preprocessing for RUSLAN corpus [Russian TTS]

.gitignore

@@ -3,3 +3,7 @@ __pycache__/
 *.pyc
 .DS_Store
 run*.sh
+*.wav
+Ruslan/
+training/*
+checkpoints/

README.md

@@ -71,6 +71,7 @@ Pull requests are welcome!
    The following are supported out of the box:
     * [LJ Speech](https://keithito.com/LJ-Speech-Dataset/) (Public Domain)
     * [Blizzard 2012](http://www.cstr.ed.ac.uk/projects/blizzard/2012/phase_one) (Creative Commons Attribution Share-Alike)
+    * [RUSLAN](https://ruslan-corpus.github.io/) (Creative Commons Attribution Share-Alike)
 
    You can use other datasets if you convert them to the right format. See [TRAINING_DATA.md](TRAINING_DATA.md) for more info.
 
@@ -98,12 +99,20 @@ Pull requests are welcome!
              |- lab
              |- wav
    ```
+   or like this for RUSLAN corpus:
+   ```
+   tacotron
+      |-Ruslan
+         |-metadata_RUSLAN_22200.csv
+         |-wavs
+   ```
 
 3. **Preprocess the data**
    ```
-   python3 preprocess.py --dataset ljspeech
+   python3 preprocess.py --dataset ljspeech --base_dir ~/tacotron
    ```
      * Use `--dataset blizzard` for Blizzard data
+     * Or `--dataset ruslan` for Ruslan corpus
 
 4. **Train a model**
    ```
@@ -159,17 +168,17 @@ Pull requests are welcome!
     `--restore_step=150000` flag to train.py (replacing 150000 with a step number prior to the
     spike). **Update**: a recent [fix](https://github.com/keithito/tacotron/pull/7) to gradient
     clipping by @candlewill may have fixed this.
-    
+
   * During eval and training, audio length is limited to `max_iters * outputs_per_step * frame_shift_ms`
     milliseconds. With the defaults (max_iters=200, outputs_per_step=5, frame_shift_ms=12.5), this is
     12.5 seconds.
-    
+
     If your training examples are longer, you will see an error like this:
     `Incompatible shapes: [32,1340,80] vs. [32,1000,80]`
-    
+
     To fix this, you can set a larger value of `max_iters` by passing `--hparams="max_iters=300"` to
     train.py (replace "300" with a value based on how long your audio is and the formula above).
-    
+
   * Here is the expected loss curve when training on LJ Speech with the default hyperparameters:
     ![Loss curve](https://user-images.githubusercontent.com/1945356/36077599-c0513e4a-0f21-11e8-8525-07347847720c.png)
 

datasets/ruslan.py

@@ -0,0 +1,72 @@
+from concurrent.futures import ProcessPoolExecutor
+from functools import partial
+import numpy as np
+import os
+from util import audio
+
+
+def build_from_path(in_dir, out_dir, num_workers=1, tqdm=lambda x: x):
+  '''Preprocesses the Ruslan dataset from a given input path into a given output directory.
+
+    Args:
+      in_dir: The directory where you have downloaded the Ruslan dataset
+      out_dir: The directory to write the output into
+      num_workers: Optional number of worker processes to parallelize across
+      tqdm: You can optionally pass tqdm to get a nice progress bar
+
+    Returns:
+      A list of tuples describing the training examples. This should be written to train.txt
+  '''
+
+  # We use ProcessPoolExecutor to parallelize across processes. This is just an optimization and you
+  # can omit it and just call _process_utterance on each input if you want.
+  executor = ProcessPoolExecutor(max_workers=num_workers)
+  futures = []
+  index = 1
+  with open(os.path.join(in_dir, 'metadata_RUSLAN_22200.csv'), encoding='utf-8') as f:
+    for line in f:
+      parts = line.strip().split('|')
+      wav_file_name = parts[0]
+      text = parts[1]
+
+      wav_path = os.path.join(in_dir, 'wavs', '%s.wav' % wav_file_name)
+
+      futures.append(executor.submit(partial(_process_utterance, out_dir, index, wav_path, text)))
+      index += 1
+  return [future.result() for future in tqdm(futures)]
+
+
+def _process_utterance(out_dir, index, wav_path, text):
+  '''Preprocesses a single utterance audio/text pair.
+
+  This writes the mel and linear scale spectrograms to disk and returns a tuple to write
+  to the train.txt file.
+
+  Args:
+    out_dir: The directory to write the spectrograms into
+    index: The numeric index to use in the spectrogram filenames.
+    wav_path: Path to the audio file containing the speech input
+    text: The text spoken in the input audio file
+
+  Returns:
+    A (spectrogram_filename, mel_filename, n_frames, text) tuple to write to train.txt
+  '''
+
+  # Load the audio to a numpy array:
+  wav = audio.load_wav(wav_path)
+
+  # Compute the linear-scale spectrogram from the wav:
+  spectrogram = audio.spectrogram(wav).astype(np.float32)
+  n_frames = spectrogram.shape[1]
+
+  # Compute a mel-scale spectrogram from the wav:
+  mel_spectrogram = audio.melspectrogram(wav).astype(np.float32)
+
+  # Write the spectrograms to disk:
+  spectrogram_filename = 'ljspeech-spec-%05d.npy' % index
+  mel_filename = 'ljspeech-mel-%05d.npy' % index
+  np.save(os.path.join(out_dir, spectrogram_filename), spectrogram.T, allow_pickle=False)
+  np.save(os.path.join(out_dir, mel_filename), mel_spectrogram.T, allow_pickle=False)
+
+  # Return a tuple describing this training example:
+  return (spectrogram_filename, mel_filename, n_frames, text)

hparams.py

@@ -5,7 +5,7 @@
 hparams = tf.contrib.training.HParams(
   # Comma-separated list of cleaners to run on text prior to training and eval. For non-English
   # text, you may want to use "basic_cleaners" or "transliteration_cleaners" See TRAINING_DATA.md.
-  cleaners='english_cleaners',
+  cleaners='basic_cleaners',
 
   # Audio:
   num_mels=80,

preprocess.py

@@ -2,7 +2,7 @@
 import os
 from multiprocessing import cpu_count
 from tqdm import tqdm
-from datasets import blizzard, ljspeech
+from datasets import blizzard, ljspeech, ruslan
 from hparams import hparams
 
 
@@ -21,6 +21,13 @@ def preprocess_ljspeech(args):
   metadata = ljspeech.build_from_path(in_dir, out_dir, args.num_workers, tqdm=tqdm)
   write_metadata(metadata, out_dir)
 
+def preprocess_ruslan(args):
+  in_dir = os.path.join(args.base_dir, 'Ruslan')
+  out_dir = os.path.join(args.base_dir, args.output)
+  os.makedirs(out_dir, exist_ok=True)
+  metadata = ruslan.build_from_path(in_dir, out_dir, args.num_workers, tqdm=tqdm)
+  write_metadata(metadata, out_dir)
+
 
 def write_metadata(metadata, out_dir):
   with open(os.path.join(out_dir, 'train.txt'), 'w', encoding='utf-8') as f:
@@ -37,13 +44,15 @@ def main():
   parser = argparse.ArgumentParser()
   parser.add_argument('--base_dir', default=os.path.expanduser('~/tacotron'))
   parser.add_argument('--output', default='training')
-  parser.add_argument('--dataset', required=True, choices=['blizzard', 'ljspeech'])
+  parser.add_argument('--dataset', required=True, choices=['blizzard', 'ljspeech', 'ruslan'])
   parser.add_argument('--num_workers', type=int, default=cpu_count())
   args = parser.parse_args()
   if args.dataset == 'blizzard':
     preprocess_blizzard(args)
   elif args.dataset == 'ljspeech':
     preprocess_ljspeech(args)
+  elif args.dataset == 'ruslan':
+    preprocess_ruslan(args)
 
 
 if __name__ == "__main__":

requirements.txt

@@ -2,9 +2,9 @@
 # depends on your platform. It is assumed you have already installed tensorflow.
 falcon==1.2.0
 inflect==0.2.5
-librosa==0.5.1
-matplotlib==2.0.2
-numpy==1.14.3
-scipy==0.19.0
+librosa
+matplotlib
+numpy<1.19.0,>=1.16.0
+scipy
 tqdm==4.11.2
 Unidecode==0.4.20

text/symbols.py

@@ -8,7 +8,7 @@
 
 _pad        = '_'
 _eos        = '~'
-_characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!\'(),-.:;? '
+_characters = 'АБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдеёжзийклмнопрстуфхцчшщъыьэюяABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!\'(),-.:;? '
 
 # Prepend "@" to ARPAbet symbols to ensure uniqueness (some are the same as uppercase letters):
 _arpabet = ['@' + s for s in cmudict.valid_symbols]

util/audio.py

@@ -3,6 +3,7 @@
 import math
 import numpy as np
 import tensorflow as tf
+from scipy.io import wavfile
 import scipy
 from hparams import hparams
 
@@ -13,7 +14,7 @@ def load_wav(path):
 
 def save_wav(wav, path):
   wav *= 32767 / max(0.01, np.max(np.abs(wav)))
-  scipy.io.wavfile.write(path, hparams.sample_rate, wav.astype(np.int16))
+  wavfile.write(path, hparams.sample_rate, wav.astype(np.int16))
 
 
 def preemphasis(x):