Strip decoded text

This commit is contained in:
Maria Schreiber
2017-11-30 23:07:24 -05:00
parent 4594650aab
commit ccc07e0f07
2 changed files with 36 additions and 19 deletions

View File

@@ -2,8 +2,10 @@
import he from 'he'; import he from 'he';
import axios from 'axios'; import axios from 'axios';
import { find } from 'lodash'; import {
// import striptags from 'striptags'; find
} from 'lodash';
import striptags from 'striptags';
export async function getSubtitles({ export async function getSubtitles({
videoID, videoID,
@@ -12,7 +14,9 @@ export async function getSubtitles({
videoID: string, videoID: string,
lang: 'en', lang: 'en',
}) { }) {
const { data } = await axios.get( const {
data
} = await axios.get(
`https://youtube.com/get_video_info?video_id=${videoID}` `https://youtube.com/get_video_info?video_id=${videoID}`
); );
@@ -24,17 +28,25 @@ export async function getSubtitles({
const regex = /({"captionTracks":.*isTranslatable":(true|false)}])/; const regex = /({"captionTracks":.*isTranslatable":(true|false)}])/;
const [match] = regex.exec(decodedData); const [match] = regex.exec(decodedData);
const { captionTracks } = JSON.parse(`${match}}`); const {
captionTracks
} = JSON.parse(`${match}}`);
const subtitle = const subtitle =
find(captionTracks, { vssId: `.${lang}` }) || find(captionTracks, {
find(captionTracks, { vssId: `a.${lang}` }); vssId: `.${lang}`
}) ||
find(captionTracks, {
vssId: `a.${lang}`
});
// * ensure we have found the correct subtitle lang // * ensure we have found the correct subtitle lang
if (!subtitle || (subtitle && !subtitle.baseUrl)) if (!subtitle || (subtitle && !subtitle.baseUrl))
throw new Error(`Could not find ${lang} captions for ${videoID}`); throw new Error(`Could not find ${lang} captions for ${videoID}`);
const { data: transcript } = await axios.get(subtitle.baseUrl); const {
data: transcript
} = await axios.get(subtitle.baseUrl);
const lines = transcript const lines = transcript
.replace('<?xml version="1.0" encoding="utf-8" ?><transcript>', '') .replace('<?xml version="1.0" encoding="utf-8" ?><transcript>', '')
.replace('</transcript>', '') .replace('</transcript>', '')
@@ -44,21 +56,22 @@ export async function getSubtitles({
const startRegex = /start="([\d.]+)"/; const startRegex = /start="([\d.]+)"/;
const durRegex = /dur="([\d.]+)"/; const durRegex = /dur="([\d.]+)"/;
const fontTag = new RegExp('<'+'font'+'[^><]*>|<.'+'font'+'[^><]*>','g');
const [, start] = startRegex.exec(line); const [, start] = startRegex.exec(line);
const [, dur] = durRegex.exec(line); const [, dur] = durRegex.exec(line);
const htmlText = line const htmlText = line
.replace(/<text.+>/, '') .replace(/<text.+>/, '')
.replace(/&amp;/gi, '&') .replace(/&amp;/gi, '&')
.replace(/<\/?[^>]+(>|$)/g, '') .replace(/<\/?[^>]+(>|$)/g, '');
.replace(fontTag, '');
// const strippedText = striptags(htmlText); const decodedText = he.decode(htmlText);
const text = striptags(decodedText);
const text = he.decode(htmlText); return {
start,
return { start, dur, text }; dur,
text
};
}); });
return lines; return lines;

View File

@@ -2956,6 +2956,10 @@ strip-json-comments@~2.0.1:
version "2.0.1" version "2.0.1"
resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a"
striptags@^3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/striptags/-/striptags-3.1.0.tgz#763e534338d9cf542f004a4b1eb099e32d295e44"
supports-color@^2.0.0: supports-color@^2.0.0:
version "2.0.0" version "2.0.0"
resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7"