diff --git a/src/transformers/models/seamless_m4t/modeling_seamless_m4t.py b/src/transformers/models/seamless_m4t/modeling_seamless_m4t.py index f619dd9e7..c0fe60a64 100755 --- a/src/transformers/models/seamless_m4t/modeling_seamless_m4t.py +++ b/src/transformers/models/seamless_m4t/modeling_seamless_m4t.py @@ -3496,7 +3496,6 @@ class SeamlessM4TForTextToSpeech(SeamlessM4TPreTrainedModel): self.device ) kwargs_speech["decoder_input_ids"] = t2u_decoder_input_ids - # second generation unit_ids = self.t2u_model.generate(inputs_embeds=t2u_input_embeds, **kwargs_speech) output_unit_ids = unit_ids.detach().clone() diff --git a/src/transformers/pipelines/pt_utils.py b/src/transformers/pipelines/pt_utils.py index c39f906f6..652d1eb54 100644 --- a/src/transformers/pipelines/pt_utils.py +++ b/src/transformers/pipelines/pt_utils.py @@ -128,9 +128,12 @@ class PipelineIterator(IterableDataset): # Try to infer the size of the batch if isinstance(processed, torch.Tensor): first_tensor = processed + elif isinstance(processed, tuple): + first_tensor = processed[0] else: key = list(processed.keys())[0] first_tensor = processed[key] + if isinstance(first_tensor, list): observed_batch_size = len(first_tensor) else: @@ -140,7 +143,7 @@ class PipelineIterator(IterableDataset): # elements. self.loader_batch_size = observed_batch_size # Setting internal index to unwrap the batch - self._loader_batch_data = processed + self._loader_batch_data = processed[0] if isinstance(processed, tuple) else processed self._loader_batch_index = 0 return self.loader_batch_item() else: diff --git a/src/transformers/pipelines/text_to_audio.py b/src/transformers/pipelines/text_to_audio.py index 58c21cc12..81653f14d 100644 --- a/src/transformers/pipelines/text_to_audio.py +++ b/src/transformers/pipelines/text_to_audio.py @@ -200,7 +200,10 @@ class TextToAudioPipeline(Pipeline): def postprocess(self, waveform): output_dict = {} - + if isinstance(waveform, dict): + waveform = waveform["waveform"] + elif isinstance(waveform, tuple): + waveform = waveform[0] output_dict["audio"] = waveform.cpu().float().numpy() output_dict["sampling_rate"] = self.sampling_rate diff --git a/tests/pipelines/test_pipelines_text_to_audio.py b/tests/pipelines/test_pipelines_text_to_audio.py index a9f1eccae..b780d26d7 100644 --- a/tests/pipelines/test_pipelines_text_to_audio.py +++ b/tests/pipelines/test_pipelines_text_to_audio.py @@ -66,6 +66,27 @@ class TextToAudioPipelineTests(unittest.TestCase): audio = [output["audio"] for output in outputs] self.assertEqual([ANY(np.ndarray), ANY(np.ndarray)], audio) + @slow + @require_torch + def test_medium_seamless_m4t_pt(self): + speech_generator = pipeline(task="text-to-audio", model="facebook/hf-seamless-m4t-medium", framework="pt") + + for forward_params in [{"tgt_lang": "eng"}, {"return_intermediate_token_ids": True, "tgt_lang": "eng"}]: + outputs = speech_generator("This is a test", forward_params=forward_params) + self.assertEqual({"audio": ANY(np.ndarray), "sampling_rate": 16000}, outputs) + + # test two examples side-by-side + outputs = speech_generator(["This is a test", "This is a second test"], forward_params=forward_params) + audio = [output["audio"] for output in outputs] + self.assertEqual([ANY(np.ndarray), ANY(np.ndarray)], audio) + + # test batching + outputs = speech_generator( + ["This is a test", "This is a second test"], forward_params=forward_params, batch_size=2 + ) + audio = [output["audio"] for output in outputs] + self.assertEqual([ANY(np.ndarray), ANY(np.ndarray)], audio) + @slow @require_torch def test_small_bark_pt(self):