diff --git a/pyannote/audio/pipelines/__init__.py b/pyannote/audio/pipelines/__init__.py index 0c7d2f25..475fa17f 100644 --- a/pyannote/audio/pipelines/__init__.py +++ b/pyannote/audio/pipelines/__init__.py @@ -24,6 +24,7 @@ from .multilabel import MultiLabelSegmentation from .overlapped_speech_detection import OverlappedSpeechDetection from .resegmentation import Resegmentation from .speaker_diarization import SpeakerDiarization +from .speaker_separation import JointSpeakerDiarizationAndSeparation from .voice_activity_detection import VoiceActivityDetection __all__ = [ @@ -32,4 +33,5 @@ __all__ = [ "SpeakerDiarization", "Resegmentation", "MultiLabelSegmentation", + "JointSpeakerDiarizationAndSeparation", ] diff --git a/pyannote/audio/pipelines/speaker_separation.py b/pyannote/audio/pipelines/speaker_separation.py index 6580530c..4cc9a14c 100644 --- a/pyannote/audio/pipelines/speaker_separation.py +++ b/pyannote/audio/pipelines/speaker_separation.py @@ -415,6 +415,7 @@ class JointSpeakerDiarizationAndSeparation(SpeakerDiarizationMixin, Pipeline): min_speakers: int = None, max_speakers: int = None, hook: Optional[Callable] = None, + zero: bool = False, ) -> Annotation: """Apply speaker diarization @@ -506,16 +507,6 @@ class JointSpeakerDiarizationAndSeparation(SpeakerDiarizationMixin, Pipeline): hard_clusters[inactive_speakers] = -2 - clustered_separations = self.reconstruct(separations, hard_clusters) - sources = Inference.aggregate( - clustered_separations, - frames=self._separation_frames, - hamming=True, - missing=0.0, - skip_average=True, - ) - # shape is (num_frames, num_speakers) - clustered_segmentations = self.reconstruct(segmentations, hard_clusters) discrete_diarization = self.to_diarization(clustered_segmentations, count) hook("discrete_diarization", discrete_diarization) @@ -528,6 +519,21 @@ class JointSpeakerDiarizationAndSeparation(SpeakerDiarizationMixin, Pipeline): ) diarization.uri = file["uri"] + clustered_separations = self.reconstruct(separations, hard_clusters) + sources = Inference.aggregate( + clustered_separations, + frames=self._separation_frames, + hamming=True, + missing=0.0, + skip_average=True, + ) + # shape is (num_frames, num_speakers) + + # zero-out sources when speaker is inactive + # WARNING: this should be rewritten to avoid huge memory consumption + if zero: + sources.data = sources.data * discrete_diarization.align(sources).data + # when reference is available, use it to map hypothesized speakers # to reference speakers (this makes later error analysis easier # but does not modify the actual output of the diarization pipeline)