python3 compatibility (no more python2 support)

abkhazia/acoustic/__init__.py

@@ -46,7 +46,7 @@ class forward the parameters of the Kaldi scripts it relies on in an
 .. python
 
    am_tri = Triphone(corpus, lm_dir, am_mono_dir, am_tri_dir)
-   for name, entry in am_mono.options.iteritems():
+   for name, entry in am_mono.options.items():
        print name, entry.value, entry.help, entry.default, entry.type
 
 

abkhazia/acoustic/monophone.py

@@ -167,7 +167,7 @@ def create(self):
         split_ali = {}
         for job in range(1, self.njobs + 1):
             split_ali[job] = []
-        for utt, line in ali.iteritems():
+        for utt, line in ali.items():
             try:
                 job = split_utt[utt]
                 split_ali[job].append(utt + ' ' + line.strip())

abkhazia/acoustic/neural_network.py

@@ -174,7 +174,7 @@ def _train_pnorm_fast(self):
         # that need further preprocessing/formatting (see above)
         nnet_opts = ' '.join(
             '--{} {}'.format(k, v.value)
-            for k, v in self.options.iteritems()
+            for k, v in self.options.items()
             if k not in self._njobs_options and k not in self._egs_options)
 
         # convert the max_high_io_jobs option to what kaldi expect...
@@ -193,7 +193,7 @@ def _train_pnorm_fast(self):
 
         egs_opts = '--egs-opts "{}"'.format(
             ' '.join('--{} {}'.format(k, v.value)
-                     for k, v in self.options.iteritems()
+                     for k, v in self.options.items()
                      if k in self._egs_options))
 
         # feeding the --cmd option

abkhazia/align/__init__.py

@@ -13,4 +13,4 @@
 # You should have received a copy of the GNU General Public License
 # along with abkhazia. If not, see <http://www.gnu.org/licenses/>.
 
-from align import Align
+from abkhazia.align.align import Align

abkhazia/align/align.py

@@ -256,7 +256,7 @@ def _read_result_utts(self, start):
         for _file in sorted(
                 [os.path.join(path, f) for f in os.listdir(path)
                  if f.startswith(start)]):
-            data += gzip.open(_file, 'r').readlines()
+            data += [l.decode() for l in gzip.open(_file, 'r')]
 
         return {line[0]: ' '.join(line[1:]) for line in
                 (l.replace('[', '').replace(']', '').split() for l in data)}
@@ -267,7 +267,7 @@ def _read_alignment(
             first_frame_center_time=.0125,
             frame_width=0.025, frame_spacing=0.01):
         """Tokenize raw kaldi alignment output"""
-        for utt_id, line in ali.iteritems():
+        for utt_id, line in ali.items():
             start = first_frame_center_time - frame_width / 2.0
             current_frame_center_time = first_frame_center_time
 
@@ -356,11 +356,11 @@ def _export_phones_and_words(self, int2phone, ali, post):
 
         # text[utt_id] = list of words
         text = {k: v.strip().split()
-                for k, v in self.corpus.text.iteritems()}
+                for k, v in self.corpus.text.items()}
 
         # lexicon[word] = list of phones
         lexicon = {k: v.strip().split()
-                   for k, v in self.corpus.lexicon.iteritems()}
+                   for k, v in self.corpus.lexicon.items()}
 
         words = []
 
@@ -372,12 +372,13 @@ def _export_phones_and_words(self, int2phone, ali, post):
         word_pos = defaultdict(list)
 
         # create list of all the phones using the lexicon
-        for word in text[utt_id]:
-            try:
-                list_phones[utt_id] += lexicon[word]
-                word_pos[utt_id] += [word] * len(lexicon[word])
-            except KeyError:
-                continue
+        for utt_id in self.corpus.utts():
+            for word in text[utt_id]:
+                try:
+                    list_phones[utt_id] += lexicon[word]
+                    word_pos[utt_id] += [word] * len(lexicon[word])
+                except KeyError:
+                    continue
 
         try:
             for utt_id, utt_align in self._read_utts(phones):
@@ -426,7 +427,7 @@ def phone_word_dtw(self, utt_align, text):
         """ Get the word level alignment from the phone level
             alignment and the lexicon """
         lexicon = {k: v.strip().split()
-                   for k, v in self.corpus.lexicon.iteritems()}
+                   for k, v in self.corpus.lexicon.items()}
 
         list_phones = []
         word_pos = []
@@ -571,7 +572,7 @@ def convert_to_word_position_dependent(alignment):
     """Add the position dependent suffixes to phones in `alignment`"""
     wpd_alignment = {}
 
-    for utt, utt_phones in alignment.iteritems():
+    for utt, utt_phones in alignment.items():
         utt_phones_wpd = []
         # iterate by words
         for word in yield_on_words(utt_phones):

abkhazia/commands/abkhazia_acoustic.py

@@ -139,7 +139,7 @@ def run(cls, args):
             recipe.delete_recipe = False
 
         # setup the model options parsed from command line
-        for k, v in vars(args).iteritems():
+        for k, v in vars(args).items():
             try:
                 recipe.set_option(k.replace('_', '-'), v)
             except KeyError:

abkhazia/commands/abkhazia_decode.py

@@ -103,7 +103,7 @@ def run(cls, args):
         recipe.delete_recipe = False if args.recipe else True
 
         # setup the model options parsed from command line
-        for k, v in vars(args).iteritems():
+        for k, v in vars(args).items():
             try:
                 recipe.set_option(k.replace('_', '-'), v)
             except KeyError:

abkhazia/commands/abkhazia_filter.py

@@ -32,36 +32,38 @@ def add_parser(cls, subparsers):
         parser, _ = super(AbkhaziaFilter, cls).add_parser(subparsers)
 
         group = parser.add_argument_group('filter arguments')
-    
+
         group.add_argument(
             '-f', '--function', type=str, metavar='<filter>',
             help='''Specifies the filtering function used on the speech distribution.
             Options are : power-law, step, exponential...''')
 
         group.add_argument(
-            '-n','--nb_speaker',default=None,type=int,metavar='<distribution>',
+            '-n', '--nb_speaker', default=None, type=int,
+            metavar='<distribution>',
             help='''An integer >1, represents the number of speaker we will keep
-                to plot the distribution (if no number specified, we keep all of
-                them)''')
-           
+            to plot the distribution (if no number specified, we keep all of
+            them)''')
+
         group.add_argument(
-            '-ns','--new_speakers',type=int,help=''' and integer >1, represents 
+            '-ns', '--new_speakers', type=int,
+            help=''' and integer >1, represents
             the number of speakers to include in the "family" part''')
-        
+
         group.add_argument(
-            '--plot',action="store_true",
+            '--plot', action="store_true",
             help='''If plot==True, a plot of the speech duration distribution and
             of the filtering function will be displayed''')
-        
+
         group.add_argument(
-            '--trim',action="store_true",
-            help='''if trim==True, the unwanted utterances will be removed from the 
+            '--trim', action="store_true",
+            help='''if trim==True, the unwanted utterances will be removed from the
             wav_files, and the segments updated accordingly. If trim==False,
             the segments file, text file, and utt2spk file will be updated,
             but the wav will still contain the unwanted utterances.''')
         group.add_argument(
-            '--THCHS30', action='store_true', 
-            help='''Set to true if treating the THCHS30 corpus, to avoid 
+            '--THCHS30', action='store_true',
+            help='''Set to true if treating the THCHS30 corpus, to avoid
             repetition of text between speakers.''')
 
         return parser
@@ -75,23 +77,22 @@ def run(cls, args):
         corpus = Corpus.load(corpus_dir, validate=args.validate, log=log)
 
         # retrieve the test proportion
-        (subcorpus,not_kept_utterances)=corpus.create_filter(
-                output_dir, 
+        (subcorpus, not_kept_utterances) = corpus.create_filter(
+                output_dir,
                 function=args.function,
                 nb_speaker=args.nb_speaker,
                 new_speakers=args.new_speakers,
                 THCHS30=args.THCHS30)
-        
-        if args.plot: 
+
+        if args.plot:
             subcorpus.plot()
 
         if args.trim:
-            print "trimming utterances"
+            print("trimming utterances")
             subcorpus.save(
                     os.path.join(
                         output_dir, args.function, 'data'),
                     no_wavs=True, copy_wavs=False)
             subcorpus.trim(
                     corpus_dir, output_dir,
                     args.function, not_kept_utterances)
-

abkhazia/commands/abkhazia_main.py

@@ -83,7 +83,7 @@ def __call__(self, parser, namespace, value, option_string=None):
                     raise IOError(
                         'configuration file not found {}'.format(value))
 
-                print 'loading configuration from {}'.format(value)
+                print('loading configuration from {}'.format(value))
                 utils.config.read(value)
 
         # add a configuration argument

abkhazia/commands/abkhazia_prepare.py

@@ -158,7 +158,7 @@ def _output_dir(cls, args):
             else args.output_dir, 'data')
 
         if args.force and os.path.exists(output_dir):
-            print 'removing {}'.format(output_dir)
+            print('removing {}'.format(output_dir))
             shutil.rmtree(output_dir)
 
         return output_dir
@@ -536,7 +536,7 @@ def describe_corpora(cls):
             # key length is len('librispeech ') == 11
             key + ' '*(11 - len(key)),
             value.preparator.description)
-                for key, value in sorted(cls.supported_corpora.iteritems())]
+                for key, value in sorted(cls.supported_corpora.items())]
 
     @classmethod
     def add_parser(cls, subparsers):
@@ -555,7 +555,7 @@ def add_parser(cls, subparsers):
             metavar='<corpus>', dest='corpus',
             help=textwrap.dedent('supported corpora are:\n' +
                                  '\n'.join(cls.describe_corpora())))
-        for corpus in cls.supported_corpora.itervalues():
+        for corpus in cls.supported_corpora.values():
             corpus.add_parser(subparsers)
 
     @classmethod

abkhazia/commands/abstract_command.py

@@ -167,7 +167,7 @@ def _parse_output_dir(cls, output, corpus, name=None, force=False):
 
         # if --force, remove any existing output_dir
         if force and os.path.exists(output):
-            print 'overwriting {}'.format(output)
+            print('overwriting {}'.format(output))
             shutil.rmtree(output)
 
         return output

abkhazia/corpus/corpus.py

@@ -170,11 +170,11 @@ def is_valid(self, njobs=utils.default_njobs()):
 
     def utts(self):
         """Return the list of utterance ids stored in the corpus"""
-        return self.utt2spk.keys()
+        return list(self.utt2spk.keys())
 
     def spks(self):
         """Return the list of speaker ids stored in the corpus"""
-        return self.spk2utt().keys()
+        return list(self.spk2utt().keys())
 
     def spk2utt(self):
         """Return a dict of speakers mapped to an utterances list
@@ -185,10 +185,10 @@ def spk2utt(self):
 
         """
         # init an empty list for all speakers
-        spk2utt = {spk: [] for spk in set(self.utt2spk.itervalues())}
+        spk2utt = {spk: [] for spk in set(self.utt2spk.values())}
 
         # populate lists with utterance ids
-        for utt, spk in self.utt2spk.iteritems():
+        for utt, spk in self.utt2spk.items():
             spk2utt[spk].append(utt)
         return spk2utt
 
@@ -206,7 +206,7 @@ def _float(t):
             return None if t is None else float(t)
 
         # populate lists with utterance ids and timestamps
-        for utt, (wav, tstart, tend) in self.segments.iteritems():
+        for utt, (wav, tstart, tend) in self.segments.items():
             wav2utt[wav].append((utt, _float(tstart), _float(tend)))
         return wav2utt
 
@@ -217,7 +217,7 @@ def utt2duration(self):
 
         """
         utt2dur = dict()
-        for utt, (wav, start, stop) in self.segments.iteritems():
+        for utt, (wav, start, stop) in self.segments.items():
             start = 0 if start is None else start
             wav_path = os.path.join(self.wav_folder, wav)
             stop = utils.wav.duration(wav_path) if stop is None else stop
@@ -234,7 +234,7 @@ def duration(self, format='seconds'):
         Raise IOError if `format` is not 'seconds' or 'datetime'
 
         """
-        total = sum(self.utt2duration().itervalues())
+        total = sum(self.utt2duration().values())
         if format == 'seconds':
             return total
         elif format == 'datetime':
@@ -253,12 +253,12 @@ def words(self, in_lexicon=True):
 
         """
         return set(
-            word for utt in self.text.itervalues() for word in utt.split()
+            word for utt in self.text.values() for word in utt.split()
             if (word in self.lexicon if in_lexicon else True))
 
     def has_several_utts_per_wav(self):
         """Return True if there is several utterances in at least one wav"""
-        for _, start, stop in self.segments.itervalues():
+        for _, start, stop in self.segments.values():
             if start is not None or stop is not None:
                 return True
         return False
@@ -469,7 +469,7 @@ def phonemize_text(self):
 
         """
         phonemized = dict()
-        for utt_id, text in self.text.iteritems():
+        for utt_id, text in self.text.items():
             phones = []
             for word in text.split():
                 try:
@@ -488,7 +488,7 @@ def plot(self):
         # connection without X forward)
         import matplotlib.pyplot as plt
 
-        utt_ids, utt_speakers = zip(*self.utt2spk.iteritems())
+        utt_ids, utt_speakers = zip(*self.utt2spk.items())
         utt2dur = self.utt2duration()
         spkr2dur = dict()
 
@@ -497,7 +497,7 @@ def plot(self):
                 [utt2dur[utt_id] for utt_id in self.utt2spk
                  if self.utt2spk[utt_id] == spkr])
 
-        sorted_speaker = sorted(spkr2dur.iteritems(), key=lambda(k, v): (v, k))
+        sorted_speaker = sorted(spkr2dur.items(), key=lambda k, v: (v, k))
         sorted_speaker.reverse()
 
         # Set plot parameters