MultiModal

#Multimodal API samples

Identify the languages of an audio file

This example shows the use of the /multimodal/speech/detectLanguage api to identify the languages of an audio file.

Demo

Source code

jQuery(function($) {
  'use strict';

  var speechLidAnalyzing = $('#speechLidAnalyzing');
  var speechLidResult = $('#speechLidResult');
  var audioFileName = $('#speechLidAudioSource').attr('src');
  var audioFile;

  function launchSpeechLid() {
    speechLidAnalyzing.removeClass('hidden');
    speechLidResult.addClass('hidden');

    var data = new FormData();
    data.append('audioFile', audioFile, audioFileName);

    $.ajax({
      method:'POST',
      url: 'https://api-platform.systran.net/multimodal/speech/detectLanguage?key=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx',
      data: data,
      cache: false,
      contentType: false,
      processData: false,
      success: function(data) {
        speechLidAnalyzing.addClass('hidden');

        if(!data || !data.speakers || !data.speakers.length)
          return;

        var html = 'Detected language';
        html += (data.speakers.length > 1) ? 's: ' : ': ';

        $.each(data.speakers, function(index, speaker) {
          if(index > 0) {
            html += ', ';
          }
          html += speaker.lang;
        });

        speechLidResult.html(html);
        speechLidResult.removeClass('hidden');
      },
      error: function(xhr, status, err) {
        console.log('Error while analyzing audio file:', err);
        speechLidAnalyzing.addClass('hidden');
        speechLidResult.removeClass('hidden');
      }
    });
  }

  //Load audio file content to be sent with the request
  var xhr = new XMLHttpRequest();
  xhr.open('GET', audioFileName, true);
  xhr.responseType = 'arraybuffer';
  xhr.onload = function(e) {
    var fileContent = new Uint8Array(this.response);
    audioFile = new Blob([fileContent], { type: "audio/mpeg"});

    $('#speechLidButton').click(launchSpeechLid);
  };
  xhr.send();
});
<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="utf-8">
    <title>Speech demo</title>
    <link href="speech-lid.css" rel="stylesheet">
    <script type="text/javascript" src="https://code.jquery.com/jquery-1.11.3.min.js"></script>
    <script type="text/javascript" src="speech-lid.js"></script>
</head>
<body>
    <div class="controls">
        <audio controls>
            <source id="speechLidAudioSource" src="speech-lid.mp3" type="audio/mpeg">
            Your browser does not support the audio element.
        </audio>

        <button type="button" id="speechLidButton" data-t>Analyze</button>
        <span id="speechLidResult" class="hidden"></span>
        <span id="speechLidAnalyzing" class="hidden" data-t>Analyzing...</span>
    </div>
</body>
</html>
#speechLidAnalyzing {
    color: #428bca;
    font-style: italic;
    margin-left: 15px;
}

#speechLidResult {
    font-style: italic;
    margin-left: 15px;
    font-weight: bold;
}

#speechLidAnalyzing.hidden {
    display:none;
}

.controls * {
    vertical-align: top;
}

Get text from an audio file

This example shows the use of the /multimodal/speech/transcribe api to get a text from an audio file.

Demo

Source code

jQuery(function($) {
  'use strict';

  var transcribing = $('#transcribing');
  var audioFileName = $('#audioSource').attr('src');
  var audioFile;

  function launchSpeechTranscription() {
    transcribing.removeClass('hidden');
    $('#transcription').html('');

    var data = new FormData();
    data.append('audioFile', audioFile, audioFileName);

    $.ajax({
      method:'POST',
      url: 'https://api-platform.systran.net/multimodal/speech/transcribe?key=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx&lang=en',
      data: data,
      cache: false,
      contentType: false,
      processData: false,
      success: function(data) {
        transcribing.addClass('hidden');

        if(!data || !data.segments || !data.segments.length)
          return;

        var html = '';

        $.each(data.segments, function(segmentIndex, segment) {
          if(!segment.words || !segment.words.length)
            return;

          html += '<p>';
          $.each(segment.words, function(wordIndex, word) {
            html += word.text;
          });
          html += '</p>';
        });

        $('#transcription').html(html);
      },
      error: function(xhr, status, err) {
        console.log('Error while transcribing audio file:', err);
        transcribing.addClass('hidden');
      }
    });
  }

  //Load audio file content to be sent with the request
  var xhr = new XMLHttpRequest();
  xhr.open('GET', audioFileName, true);
  xhr.responseType = 'arraybuffer';
  xhr.onload = function(e) {
    var fileContent = new Uint8Array(this.response);
    audioFile = new Blob([fileContent], { type: "audio/mpeg"});

    $('#transcribeButton').click(launchSpeechTranscription);
  };
  xhr.send();
});
<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="utf-8">
    <title>Speech demo</title>
    <link href="speech-transcribe.css" rel="stylesheet">
    <script type="text/javascript" src="https://code.jquery.com/jquery-1.11.3.min.js"></script>
    <script type="text/javascript" src="speech-transcribe.js"></script>
</head>
<body>
    <div class="controls">
        <audio controls>
            <source id="audioSource" src="speech-transcribe.mp3" type="audio/mpeg">
            Your browser does not support the audio element.
        </audio>

        <button type="button" id="transcribeButton" data-t>Transcribe</button>
        <div id="transcribing" class="hidden"data-t>Transcribing...</div>
    </div>
    <div id="transcription" />
</body>
</html>
#transcribing {
    color: #428bca;
    font-style: italic;
    display: inline-block;
    margin-left: 15px;
}

#transcribing.hidden {
    display:none;
}

.controls * {
    vertical-align: top;
}

#transcription {
    overflow: auto;
    height: 100px;
    font-family: monospace;
    font-size: 14px;
    background: whitesmoke;
    padding: 10px;
    margin-top: 10px;
}

Align a text with an audio file

This example shows the use of the /multimodal/speech/align api to align a text with an audio file.

Demo

Source code

jQuery(function($) {
  'use strict';

  var aligning = $('#aligning');
  var audioFileName = $('#audioSource').attr('src');
  var audioFile;
  var textFile = new Blob([$('#textSource').text()], {type : 'text/html'});

  function launchSpeechAlignment() {
    aligning.removeClass('hidden');
    $('#alignmentResults').html('');

    var data = new FormData();
    data.append('audioFile', audioFile, audioFileName);
    data.append('textFile', textFile);

    $.ajax({
      method:'POST',
      url: 'https://api-platform.systran.net/multimodal/speech/align?key=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx&lang=en',
      data: data,
      cache: false,
      contentType: false,
      processData: false,
      success: function(data) {
        aligning.addClass('hidden');

        if(!data || !data.segments || !data.segments.length)
          return;

        var html = '';

        $.each(data.segments, function(segmentIndex, segment) {
          if(!segment.words || !segment.words.length)
            return;

          var segmentLength = parseFloat(segment.end);
          html += '<div class="segment">';
          html += '<span class="time" style="left: 100%;">' + segment.end + "</span>";

          $.each(segment.words, function(wordIndex, word) {
            var wordDuration = parseFloat(word.duration);

            if(!isNaN(wordDuration) && wordDuration > 0) {
              var leftMargin = 100 * parseFloat(word.start) / segmentLength;
              var width = 100 * parseFloat(word.duration) / segmentLength;

              html += '<span class="time" style="left: ' + leftMargin + '%;">' + word.start + "</span>";
              html += '<span class="word" style="width:' + width + '%; left:' + leftMargin + '%;">' + word.text + "</span>";
            }
          });
          html += '</div>';
        });

        $('#alignmentResults').html(html);
      },
      error: function(xhr, status, err) {
        console.log('Error while aligning text with audio file:', err);
        aligning.addClass('hidden');
      }
    });
  }

  //Load audio file content to be sent with the request
  var xhr = new XMLHttpRequest();
  xhr.open('GET', audioFileName, true);
  xhr.responseType = 'arraybuffer';
  xhr.onload = function(e) {
    var fileContent = new Uint8Array(this.response);
    audioFile = new Blob([fileContent], { type: "audio/mpeg"});

    $('#alignButton').click(launchSpeechAlignment);
  };
  xhr.send();
});
<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="utf-8">
    <title>Speech demo</title>
    <link href="speech-align.css" rel="stylesheet">
    <script type="text/javascript" src="https://code.jquery.com/jquery-1.11.3.min.js"></script>
    <script type="text/javascript" src="speech-align.js"></script>
<script src="/components/iframe-resizer/2.8.8/iframeResizer.contentWindow.min.js"></script></head>
<body>
    <table id="inputs">
        <tr>
            <td>
                <h4 data-t>Audio</h4>
                <audio controls>
                    <source id="audioSource" src="speech-align.mp3" type="audio/mpeg">
                    Your browser does not support the audio element.
                </audio>
            </td>
            <td id="alignButtonWrapper">
                <button type="button" id="alignButton" data-t>Align text with audio</button>
                <div id="aligning" class="hidden" data-t>Aligning...</div>
            </td>
            <td>
                <h4>Text</h4>
                <div id="textSource" data-t>This is a test</div>
            </td>
        </tr>
    </table>

    <div id="alignmentResults" />
</body>
</html>
table#inputs {
    width: 100%;
}

table#inputs td {
    text-align: center;
    width: 40%;
    vertical-align: top;
}

table#inputs td#alignButtonWrapper {
    width: 80px;
}

table#inputs h4 {
    margin-top: 0px;
}

#alignButton {
    height: 40px;
}

#aligning {
    color: #428bca;
    font-style: italic;
    margin-top: 10px;
}

#aligning.hidden {
    display:none;
}

#alignmentResults {
    font-family: monospace;
    font-size: 14px;
    background: whitesmoke;
    margin-top: 10px;
    padding: 10px 40px 30px 10px;
    min-height: 40px;
}

.segment {
    border: 1px solid #aaa;
    position: relative;
    margin-top: 15px;
    height: 25px;
    box-shadow: 5px 5px 8px #888;
}

.word {
    background-color: #8FE7FF;
    border-left: 1px solid #00a2ff;
    border-right: 1px solid #00a2ff;
    position: absolute;
    text-align: center;
    height: 25px;
    line-height: 25px;
    font-weight: bold;
}

.time {
    border-left: 1px solid #aaa;
    position: absolute;
    text-align: center;
    height: 20px;
    top: -20px;
    padding-left: 5px;
}