diff --git a/.gitignore b/.gitignore index 7e377245..96421d6e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,7 @@ __pycache__/ *.pyc .DS_Store run*.sh +*.wav +Ruslan/ +training/* +checkpoints/ diff --git a/README.md b/README.md index c66fa5f6..e638f8d9 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ Pull requests are welcome! The following are supported out of the box: * [LJ Speech](https://keithito.com/LJ-Speech-Dataset/) (Public Domain) * [Blizzard 2012](http://www.cstr.ed.ac.uk/projects/blizzard/2012/phase_one) (Creative Commons Attribution Share-Alike) + * [RUSLAN](https://ruslan-corpus.github.io/) (Creative Commons Attribution Share-Alike) You can use other datasets if you convert them to the right format. See [TRAINING_DATA.md](TRAINING_DATA.md) for more info. @@ -98,12 +99,20 @@ Pull requests are welcome! |- lab |- wav ``` + or like this for RUSLAN corpus: + ``` + tacotron + |-Ruslan + |-metadata_RUSLAN_22200.csv + |-wavs + ``` 3. **Preprocess the data** ``` - python3 preprocess.py --dataset ljspeech + python3 preprocess.py --dataset ljspeech --base_dir ~/tacotron ``` * Use `--dataset blizzard` for Blizzard data + * Or `--dataset ruslan` for Ruslan corpus 4. **Train a model** ``` @@ -159,17 +168,17 @@ Pull requests are welcome! `--restore_step=150000` flag to train.py (replacing 150000 with a step number prior to the spike). **Update**: a recent [fix](https://github.com/keithito/tacotron/pull/7) to gradient clipping by @candlewill may have fixed this. - + * During eval and training, audio length is limited to `max_iters * outputs_per_step * frame_shift_ms` milliseconds. With the defaults (max_iters=200, outputs_per_step=5, frame_shift_ms=12.5), this is 12.5 seconds. - + If your training examples are longer, you will see an error like this: `Incompatible shapes: [32,1340,80] vs. [32,1000,80]` - + To fix this, you can set a larger value of `max_iters` by passing `--hparams="max_iters=300"` to train.py (replace "300" with a value based on how long your audio is and the formula above). - + * Here is the expected loss curve when training on LJ Speech with the default hyperparameters: ![Loss curve](https://user-images.githubusercontent.com/1945356/36077599-c0513e4a-0f21-11e8-8525-07347847720c.png) diff --git a/datasets/ruslan.py b/datasets/ruslan.py new file mode 100644 index 00000000..b54e010b --- /dev/null +++ b/datasets/ruslan.py @@ -0,0 +1,72 @@ +from concurrent.futures import ProcessPoolExecutor +from functools import partial +import numpy as np +import os +from util import audio + + +def build_from_path(in_dir, out_dir, num_workers=1, tqdm=lambda x: x): + '''Preprocesses the Ruslan dataset from a given input path into a given output directory. + + Args: + in_dir: The directory where you have downloaded the Ruslan dataset + out_dir: The directory to write the output into + num_workers: Optional number of worker processes to parallelize across + tqdm: You can optionally pass tqdm to get a nice progress bar + + Returns: + A list of tuples describing the training examples. This should be written to train.txt + ''' + + # We use ProcessPoolExecutor to parallelize across processes. This is just an optimization and you + # can omit it and just call _process_utterance on each input if you want. + executor = ProcessPoolExecutor(max_workers=num_workers) + futures = [] + index = 1 + with open(os.path.join(in_dir, 'metadata_RUSLAN_22200.csv'), encoding='utf-8') as f: + for line in f: + parts = line.strip().split('|') + wav_file_name = parts[0] + text = parts[1] + + wav_path = os.path.join(in_dir, 'wavs', '%s.wav' % wav_file_name) + + futures.append(executor.submit(partial(_process_utterance, out_dir, index, wav_path, text))) + index += 1 + return [future.result() for future in tqdm(futures)] + + +def _process_utterance(out_dir, index, wav_path, text): + '''Preprocesses a single utterance audio/text pair. + + This writes the mel and linear scale spectrograms to disk and returns a tuple to write + to the train.txt file. + + Args: + out_dir: The directory to write the spectrograms into + index: The numeric index to use in the spectrogram filenames. + wav_path: Path to the audio file containing the speech input + text: The text spoken in the input audio file + + Returns: + A (spectrogram_filename, mel_filename, n_frames, text) tuple to write to train.txt + ''' + + # Load the audio to a numpy array: + wav = audio.load_wav(wav_path) + + # Compute the linear-scale spectrogram from the wav: + spectrogram = audio.spectrogram(wav).astype(np.float32) + n_frames = spectrogram.shape[1] + + # Compute a mel-scale spectrogram from the wav: + mel_spectrogram = audio.melspectrogram(wav).astype(np.float32) + + # Write the spectrograms to disk: + spectrogram_filename = 'ljspeech-spec-%05d.npy' % index + mel_filename = 'ljspeech-mel-%05d.npy' % index + np.save(os.path.join(out_dir, spectrogram_filename), spectrogram.T, allow_pickle=False) + np.save(os.path.join(out_dir, mel_filename), mel_spectrogram.T, allow_pickle=False) + + # Return a tuple describing this training example: + return (spectrogram_filename, mel_filename, n_frames, text) diff --git a/hparams.py b/hparams.py index 05ae82a3..65402430 100644 --- a/hparams.py +++ b/hparams.py @@ -5,7 +5,7 @@ hparams = tf.contrib.training.HParams( # Comma-separated list of cleaners to run on text prior to training and eval. For non-English # text, you may want to use "basic_cleaners" or "transliteration_cleaners" See TRAINING_DATA.md. - cleaners='english_cleaners', + cleaners='basic_cleaners', # Audio: num_mels=80, diff --git a/preprocess.py b/preprocess.py index fddb0b8d..2a60cdcd 100644 --- a/preprocess.py +++ b/preprocess.py @@ -2,7 +2,7 @@ import os from multiprocessing import cpu_count from tqdm import tqdm -from datasets import blizzard, ljspeech +from datasets import blizzard, ljspeech, ruslan from hparams import hparams @@ -21,6 +21,13 @@ def preprocess_ljspeech(args): metadata = ljspeech.build_from_path(in_dir, out_dir, args.num_workers, tqdm=tqdm) write_metadata(metadata, out_dir) +def preprocess_ruslan(args): + in_dir = os.path.join(args.base_dir, 'Ruslan') + out_dir = os.path.join(args.base_dir, args.output) + os.makedirs(out_dir, exist_ok=True) + metadata = ruslan.build_from_path(in_dir, out_dir, args.num_workers, tqdm=tqdm) + write_metadata(metadata, out_dir) + def write_metadata(metadata, out_dir): with open(os.path.join(out_dir, 'train.txt'), 'w', encoding='utf-8') as f: @@ -37,13 +44,15 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument('--base_dir', default=os.path.expanduser('~/tacotron')) parser.add_argument('--output', default='training') - parser.add_argument('--dataset', required=True, choices=['blizzard', 'ljspeech']) + parser.add_argument('--dataset', required=True, choices=['blizzard', 'ljspeech', 'ruslan']) parser.add_argument('--num_workers', type=int, default=cpu_count()) args = parser.parse_args() if args.dataset == 'blizzard': preprocess_blizzard(args) elif args.dataset == 'ljspeech': preprocess_ljspeech(args) + elif args.dataset == 'ruslan': + preprocess_ruslan(args) if __name__ == "__main__": diff --git a/requirements.txt b/requirements.txt index 7588bb23..f6c43516 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,9 +2,9 @@ # depends on your platform. It is assumed you have already installed tensorflow. falcon==1.2.0 inflect==0.2.5 -librosa==0.5.1 -matplotlib==2.0.2 -numpy==1.14.3 -scipy==0.19.0 +librosa +matplotlib +numpy<1.19.0,>=1.16.0 +scipy tqdm==4.11.2 Unidecode==0.4.20 diff --git a/text/symbols.py b/text/symbols.py index 565baa3f..d0ed5612 100644 --- a/text/symbols.py +++ b/text/symbols.py @@ -8,7 +8,7 @@ _pad = '_' _eos = '~' -_characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!\'(),-.:;? ' +_characters = 'АБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдеёжзийклмнопрстуфхцчшщъыьэюяABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!\'(),-.:;? ' # Prepend "@" to ARPAbet symbols to ensure uniqueness (some are the same as uppercase letters): _arpabet = ['@' + s for s in cmudict.valid_symbols] diff --git a/util/audio.py b/util/audio.py index 1362f910..58358a79 100644 --- a/util/audio.py +++ b/util/audio.py @@ -3,6 +3,7 @@ import math import numpy as np import tensorflow as tf +from scipy.io import wavfile import scipy from hparams import hparams @@ -13,7 +14,7 @@ def load_wav(path): def save_wav(wav, path): wav *= 32767 / max(0.01, np.max(np.abs(wav))) - scipy.io.wavfile.write(path, hparams.sample_rate, wav.astype(np.int16)) + wavfile.write(path, hparams.sample_rate, wav.astype(np.int16)) def preemphasis(x):