Source: lib/cea/mp4_cea_parser.js

  1. /*! @license
  2.  * Shaka Player
  3.  * Copyright 2016 Google LLC
  4.  * SPDX-License-Identifier: Apache-2.0
  5.  */
  7. goog.provide('shaka.cea.Mp4CeaParser');
  9. goog.require('goog.asserts');
  10. goog.require('shaka.cea.CeaUtils');
  11. goog.require('shaka.cea.SeiProcessor');
  12. goog.require('shaka.log');
  13. goog.require('shaka.media.ClosedCaptionParser');
  14. goog.require('shaka.util.DataViewReader');
  15. goog.require('shaka.util.Error');
  16. goog.require('shaka.util.Mp4Parser');
  17. goog.require('shaka.util.Mp4BoxParsers');
  19. /**
  20.  * MPEG4 stream parser used for extracting 708 closed captions data.
  21.  * @implements {shaka.extern.ICeaParser}
  22.  * @export
  23.  */
  24. shaka.cea.Mp4CeaParser = class {
  25.   /** */
  26.   constructor() {
  27.     /**
  28.      * SEI data processor.
  29.      * @private
  30.      * @const {!shaka.cea.SeiProcessor}
  31.      */
  32.     this.seiProcessor_ = new shaka.cea.SeiProcessor();
  34.     /**
  35.      * Map of track id to corresponding timescale.
  36.      * @private {!Map<number, number>}
  37.      */
  38.     this.trackIdToTimescale_ = new Map();
  40.     /**
  41.      * Default sample duration, as specified by the TREX box.
  42.      * @private {number}
  43.      */
  44.     this.defaultSampleDuration_ = 0;
  46.     /**
  47.      * Default sample size, as specified by the TREX box.
  48.      * @private {number}
  49.      */
  50.     this.defaultSampleSize_ = 0;
  52.     /**
  53.      * @private {shaka.cea.Mp4CeaParser.BitstreamFormat}
  54.      */
  55.     this.bitstreamFormat_ = shaka.cea.Mp4CeaParser.BitstreamFormat.UNKNOWN;
  56.   }
  58.   /**
  59.    * Parses the init segment. Gets Default Sample Duration and Size from the
  60.    * TREX box, and constructs a map of Track IDs to timescales. Each TRAK box
  61.    * contains a track header (TKHD) containing track ID, and a media header box
  62.    * (MDHD) containing the timescale for the track
  63.    * @override
  64.    */
  65.   init(initSegment) {
  66.     const Mp4Parser = shaka.util.Mp4Parser;
  67.     const BitstreamFormat = shaka.cea.Mp4CeaParser.BitstreamFormat;
  68.     const trackIds = [];
  69.     const timescales = [];
  71.     const codecBoxParser = (box) => this.setBitstreamFormat_(box.name);
  73.     new Mp4Parser()
  74.         .box('moov', Mp4Parser.children)
  75.         .box('mvex', Mp4Parser.children)
  76.         .fullBox('trex', (box) => {
  77.           const parsedTREXBox = shaka.util.Mp4BoxParsers.parseTREX(
  78.               box.reader);
  80.           this.defaultSampleDuration_ = parsedTREXBox.defaultSampleDuration;
  81.           this.defaultSampleSize_ = parsedTREXBox.defaultSampleSize;
  82.         })
  83.         .box('trak', Mp4Parser.children)
  84.         .fullBox('tkhd', (box) => {
  85.           goog.asserts.assert(
  86.               box.version != null,
  87.               'TKHD is a full box and should have a valid version.');
  88.           const parsedTKHDBox = shaka.util.Mp4BoxParsers.parseTKHD(
  89.               box.reader, box.version);
  90.           trackIds.push(parsedTKHDBox.trackId);
  91.         })
  92.         .box('mdia', Mp4Parser.children)
  93.         .fullBox('mdhd', (box) => {
  94.           goog.asserts.assert(
  95.               box.version != null,
  96.               'MDHD is a full box and should have a valid version.');
  97.           const parsedMDHDBox = shaka.util.Mp4BoxParsers.parseMDHD(
  98.               box.reader, box.version);
  99.           timescales.push(parsedMDHDBox.timescale);
  100.         })
  101.         .box('minf', Mp4Parser.children)
  102.         .box('stbl', Mp4Parser.children)
  103.         .fullBox('stsd', Mp4Parser.sampleDescription)
  105.         // These are the various boxes that signal a codec.
  106.         .box('avc1', codecBoxParser)
  107.         .box('avc3', codecBoxParser)
  108.         .box('hev1', codecBoxParser)
  109.         .box('hvc1', codecBoxParser)
  110.         .box('dvh1', codecBoxParser)
  111.         .box('dvhe', codecBoxParser)
  113.         // This signals an encrypted sample, which we can go inside of to find
  114.         // the codec used.
  115.         .box('encv', Mp4Parser.visualSampleEntry)
  116.         .box('sinf', Mp4Parser.children)
  117.         .box('frma', (box) => {
  118.           const {codec} = shaka.util.Mp4BoxParsers.parseFRMA(box.reader);
  119.           this.setBitstreamFormat_(codec);
  120.         })
  122.         .parse(initSegment, /* partialOkay= */ true);
  124.     // At least one track should exist, and each track should have a
  125.     // corresponding Id in TKHD box, and timescale in its MDHD box
  126.     if (!trackIds.length|| !timescales.length ||
  127.       trackIds.length != timescales.length) {
  128.       throw new shaka.util.Error(
  129.           shaka.util.Error.Severity.CRITICAL,
  130.           shaka.util.Error.Category.TEXT,
  131.           shaka.util.Error.Code.INVALID_MP4_CEA);
  132.     }
  134.     if (this.bitstreamFormat_ == BitstreamFormat.UNKNOWN) {
  135.       shaka.log.alwaysWarn(
  136.           'Unable to determine bitstream format for CEA parsing!');
  137.     }
  139.     // Populate the map from track Id to timescale
  140.     trackIds.forEach((trackId, idx) => {
  141.       this.trackIdToTimescale_.set(trackId, timescales[idx]);
  142.     });
  143.   }
  145.   /**
  146.    * Parses each video segment. In fragmented MP4s, MOOF and MDAT come in
  147.    * pairs. The following logic gets the necessary info from MOOFs to parse
  148.    * MDATs (base media decode time, sample sizes/offsets/durations, etc),
  149.    * and then parses the MDAT boxes for CEA-708 packets using this information.
  150.    * CEA-708 packets are returned in the callback.
  151.    * @override
  152.    */
  153.   parse(mediaSegment) {
  154.     const Mp4Parser = shaka.util.Mp4Parser;
  155.     const BitstreamFormat = shaka.cea.Mp4CeaParser.BitstreamFormat;
  157.     if (this.bitstreamFormat_ == BitstreamFormat.UNKNOWN) {
  158.       // We don't know how to extract SEI from this.
  159.       return [];
  160.     }
  162.     /** @type {!Array<!shaka.extern.ICeaParser.CaptionPacket>} **/
  163.     const captionPackets = [];
  165.     // Fields that are found in MOOF boxes
  166.     let defaultSampleDuration = this.defaultSampleDuration_;
  167.     let defaultSampleSize = this.defaultSampleSize_;
  168.     let moofOffset = 0;
  169.     /** @type {!Array<shaka.util.ParsedTRUNBox>} */
  170.     let parsedTRUNs = [];
  171.     let baseMediaDecodeTime = null;
  172.     let timescale = shaka.cea.CeaUtils.DEFAULT_TIMESCALE_VALUE;
  174.     new Mp4Parser()
  175.         .box('moof', (box) => {
  176.           moofOffset = box.start;
  177.           // trun box parsing is reset on each moof.
  178.           parsedTRUNs = [];
  179.           Mp4Parser.children(box);
  180.         })
  181.         .box('traf', Mp4Parser.children)
  182.         .fullBox('trun', (box) => {
  183.           goog.asserts.assert(
  184.               box.version != null && box.flags!=null,
  185.               'TRUN is a full box and should have a valid version & flags.');
  187.           const parsedTRUN = shaka.util.Mp4BoxParsers.parseTRUN(
  188.               box.reader, box.version, box.flags);
  189.           parsedTRUNs.push(parsedTRUN);
  190.         })
  191.         .fullBox('tfhd', (box) => {
  192.           goog.asserts.assert(
  193.               box.flags != null,
  194.               'TFHD is a full box and should have valid flags.');
  196.           const parsedTFHD = shaka.util.Mp4BoxParsers.parseTFHD(
  197.               box.reader, box.flags);
  199.           // If specified, defaultSampleDuration and defaultSampleSize
  200.           // override the ones specified in the TREX box
  201.           defaultSampleDuration = parsedTFHD.defaultSampleDuration ||
  202.               this.defaultSampleDuration_;
  204.           defaultSampleSize = parsedTFHD.defaultSampleSize ||
  205.               this.defaultSampleSize_;
  207.           const trackId = parsedTFHD.trackId;
  209.           // Get the timescale from the track Id
  210.           if (this.trackIdToTimescale_.has(trackId)) {
  211.             timescale = this.trackIdToTimescale_.get(trackId);
  212.           }
  213.         })
  214.         .fullBox('tfdt', (box) => {
  215.           goog.asserts.assert(
  216.               box.version != null,
  217.               'TFDT is a full box and should have a valid version.');
  219.           const parsedTFDT = shaka.util.Mp4BoxParsers.parseTFDTInaccurate(
  220.               box.reader, box.version);
  222.           baseMediaDecodeTime = parsedTFDT.baseMediaDecodeTime;
  223.         })
  224.         .box('mdat', (box) => {
  225.           if (baseMediaDecodeTime === null) {
  226.             // This field should have been populated by the Base Media Decode
  227.             // Time in the tfdt box.
  228.             shaka.log.alwaysWarn(
  229.                 'Unable to find base media decode time for CEA captions!');
  230.             throw new shaka.util.Error(
  231.                 shaka.util.Error.Severity.CRITICAL,
  232.                 shaka.util.Error.Category.TEXT,
  233.                 shaka.util.Error.Code.INVALID_MP4_CEA);
  234.           }
  236.           const offset = moofOffset - box.start - 8;
  237.           this.parseMdat_(box.reader, baseMediaDecodeTime, timescale,
  238.               defaultSampleDuration, defaultSampleSize, offset, parsedTRUNs,
  239.               captionPackets);
  240.         })
  241.         .parse(mediaSegment, /* partialOkay= */ false);
  243.     return captionPackets;
  244.   }
  246.   /**
  247.    * Parse MDAT box.
  248.    * @param {!shaka.util.DataViewReader} reader
  249.    * @param {number} time
  250.    * @param {number} timescale
  251.    * @param {number} defaultSampleDuration
  252.    * @param {number} defaultSampleSize
  253.    * @param {number} offset
  254.    * @param {!Array<shaka.util.ParsedTRUNBox>} parsedTRUNs
  255.    * @param {!Array<!shaka.extern.ICeaParser.CaptionPacket>} captionPackets
  256.    * @private
  257.    */
  258.   parseMdat_(reader, time, timescale, defaultSampleDuration,
  259.       defaultSampleSize, offset, parsedTRUNs, captionPackets) {
  260.     const BitstreamFormat = shaka.cea.Mp4CeaParser.BitstreamFormat;
  261.     const CeaUtils = shaka.cea.CeaUtils;
  262.     let sampleIndex = 0;
  264.     // The fields in each ParsedTRUNSample contained in the sampleData
  265.     // array are nullable. In the case of sample data and sample duration,
  266.     // we use the defaults provided by the TREX/TFHD boxes. For sample
  267.     // composition time offset, we default to 0.
  268.     let sampleSize = defaultSampleSize;
  270.     // Combine all sample data.  This assumes that the samples described across
  271.     // multiple trun boxes are still continuous in the mdat box.
  272.     const sampleDatas = parsedTRUNs.map((t) => t.sampleData);
  273.     const sampleData = [].concat(...sampleDatas);
  275.     if (sampleData.length) {
  276.       sampleSize = sampleData[0].sampleSize || defaultSampleSize;
  277.     }
  279.     reader.skip(offset + parsedTRUNs[0].dataOffset);
  281.     while (reader.hasMoreData()) {
  282.       const naluSize = reader.readUint32();
  283.       const naluHeader = reader.readUint8();
  284.       let naluType = null;
  285.       let isSeiMessage = false;
  286.       let naluHeaderSize = 1;
  288.       goog.asserts.assert(this.bitstreamFormat_ != BitstreamFormat.UNKNOWN,
  289.           'Bitstream format should have been checked before now!');
  290.       switch (this.bitstreamFormat_) {
  291.         case BitstreamFormat.H264:
  292.           naluType = naluHeader & 0x1f;
  293.           isSeiMessage = naluType == CeaUtils.H264_NALU_TYPE_SEI;
  294.           break;
  296.         case BitstreamFormat.H265:
  297.           naluHeaderSize = 2;
  298.           reader.skip(1);
  299.           naluType = (naluHeader >> 1) & 0x3f;
  300.           isSeiMessage =
  301.               naluType == CeaUtils.H265_PREFIX_NALU_TYPE_SEI ||
  302.               naluType == CeaUtils.H265_SUFFIX_NALU_TYPE_SEI;
  303.           break;
  305.         default:
  306.           return;
  307.       }
  309.       if (isSeiMessage) {
  310.         let timeOffset = 0;
  312.         if (sampleIndex < sampleData.length) {
  313.           timeOffset = sampleData[sampleIndex].sampleCompositionTimeOffset || 0;
  314.         }
  316.         const pts = (time + timeOffset)/timescale;
  317.         for (const packet of this.seiProcessor_
  318.             .process(reader.readBytes(naluSize - naluHeaderSize))) {
  319.           captionPackets.push({
  320.             packet,
  321.             pts,
  322.           });
  323.         }
  324.       } else {
  325.         try {
  326.           reader.skip(naluSize - naluHeaderSize);
  327.         } catch (e) {
  328.           // It is necessary to ignore this error because it can break the start
  329.           // of playback even if the user does not want to see the subtitles.
  330.           break;
  331.         }
  332.       }
  333.       sampleSize -= (naluSize + 4);
  334.       if (sampleSize == 0) {
  335.         if (sampleIndex < sampleData.length) {
  336.           time += sampleData[sampleIndex].sampleDuration ||
  337.               defaultSampleDuration;
  338.         } else {
  339.           time += defaultSampleDuration;
  340.         }
  342.         sampleIndex++;
  344.         if (sampleIndex < sampleData.length) {
  345.           sampleSize = sampleData[sampleIndex].sampleSize || defaultSampleSize;
  346.         } else {
  347.           sampleSize = defaultSampleSize;
  348.         }
  349.       }
  350.     }
  351.   }
  353.   /**
  354.    * @param {string} codec A fourcc for a codec.
  355.    * @private
  356.    */
  357.   setBitstreamFormat_(codec) {
  358.     if (codec in shaka.cea.Mp4CeaParser.CodecBitstreamMap_) {
  359.       this.bitstreamFormat_ = shaka.cea.Mp4CeaParser.CodecBitstreamMap_[codec];
  360.     }
  361.   }
  362. };
  364. /** @enum {number} */
  365. shaka.cea.Mp4CeaParser.BitstreamFormat = {
  366.   UNKNOWN: 0,
  367.   H264: 1,
  368.   H265: 2,
  369. };
  371. /** @private {Object.<string, shaka.cea.Mp4CeaParser.BitstreamFormat>} */
  372. shaka.cea.Mp4CeaParser.CodecBitstreamMap_ = {
  373.   'avc1': shaka.cea.Mp4CeaParser.BitstreamFormat.H264,
  374.   'avc3': shaka.cea.Mp4CeaParser.BitstreamFormat.H264,
  375.   'hev1': shaka.cea.Mp4CeaParser.BitstreamFormat.H265,
  376.   'hvc1': shaka.cea.Mp4CeaParser.BitstreamFormat.H265,
  377.   // Dobly vision is also H265.
  378.   'dvh1': shaka.cea.Mp4CeaParser.BitstreamFormat.H265,
  379.   'dvhe': shaka.cea.Mp4CeaParser.BitstreamFormat.H265,
  380. };
  382. shaka.media.ClosedCaptionParser.registerParser('video/mp4',
  383.     () => new shaka.cea.Mp4CeaParser());