youtubedl.go 14.2 KB
Newer Older
1
package youtubedl
2
3

import (
4
	"context"
5
	"encoding/json"
6
7
	"encoding/xml"
	"errors"
8
9
10
	"fmt"
	"github.com/asticode/go-astisub"
	log "github.com/sirupsen/logrus"
11
12
13
	"gitlab.ethz.ch/chgerber/MessageComposition/src/pkg/util"
	"gitlab.ethz.ch/chgerber/annotation"
	"gitlab.ethz.ch/chgerber/monitor"
14
15
16
17
	"google.golang.org/api/googleapi/transport"
	"google.golang.org/api/option"
	"google.golang.org/api/youtube/v3"
	"net/http"
18
19
	"net/url"
	"os"
20
	"os/exec"
21
22
	"path/filepath"
	"regexp"
23
	"strconv"
24
25
26
	"strings"
)

27
28
29
30
31
32
33
34
35
36
37
// Subtitle is a wrapper type for YouTube subtitles that can be parsed by astisub
type Subtitle struct {
	*astisub.Subtitles
}

// Annotations turns the subtitles into annotations with the src field set as given
// Method of the annotation.Caption IF
func (s *Subtitle) Annotations(src string) []*annotation.Annotation {
	return annotation.SubToAnnotation(s.Subtitles, src)
}

38
// SubtitleFormat type describes a subtitle type (e.g. `vtt`)
39
40
41
42
43
44
45
type SubtitleFormat string

func (sf SubtitleFormat) String() string {
	return string(sf)
}

const (
46
47
48
49
50
51
52
53
54
	// SubVTT caption of type vtt
	SubVTT SubtitleFormat = "vtt"
	// SubSRT caption of type srt
	SubSRT SubtitleFormat = "srt"
	// SubSSA caption of type ssa
	SubSSA SubtitleFormat = "ssa"
	// SubSTL caption of type stl
	SubSTL SubtitleFormat = "stl"
	// SubTTML caption of type ttml
55
	SubTTML SubtitleFormat = "ttml"
56
57
	// SubSRV3 caption of the youtube xml timedtext type with file extension .srv3
	SubSRV3 SubtitleFormat = "srv3"
58
59
)

60
61
62
63
64
65
66
67
// SupportedSubs is a list in preferred order of subtitle types that are supported to be parsed
var SupportedSubs = []SubtitleFormat{
	//SubSRV3,
	SubSRT,
	SubTTML,
	SubSSA,
	SubSTL,
	//SubVTT TODO Timing does not work with YouTube subs of this type. -> Fix parsing
68
69
}

70
// ErrYoutubeDLLangNotSupported is thrown when language not supported
71
72
73
74
75
76
type ErrYoutubeDLLangNotSupported string

func (e ErrYoutubeDLLangNotSupported) Error() string {
	return string(e)
}

77
78
79
80
81
82
83
// ErrYoutubeVideoUnavailable is thrown when language not supported
type ErrYoutubeVideoUnavailable string

func (e ErrYoutubeVideoUnavailable) Error() string {
	return string(e)
}

84
// ErrYoutubeDL are errors with related to the youtubedl pkg
85
86
87
88
89
90
type ErrYoutubeDL string

func (e ErrYoutubeDL) Error() string {
	return string(e)
}

91
92
func runYoutubeDL(args []string) (output []byte, err error) {
	cmd := exec.Command("youtube-dl", args...)
93
	log.Debug(args)
94
	output, err = cmd.CombinedOutput()
95
	log.Trace(string(output))
96
97
	if err != nil && strings.Contains(string(output), "video is unavailable") {
		return nil, ErrYoutubeVideoUnavailable(string(output))
98
	}
99
	return output, err
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
}

func getVideoInfo(id string) (info map[string]interface{}, err error) {

	args := []string{
		"--skip-download",
		"-J",
		"https://youtu.be/" + id,
	}

	output, err := runYoutubeDL(args)
	if err != nil {
		return info, err
	}

115
116
117
118
119
120
121
	lines := strings.Split(strings.Trim(string(output), "\n\t "), "\n")

	if len(lines) == 0 {
		return nil, ErrYoutubeDL("youtube-dl response decoding error")
	}

	err = json.Unmarshal([]byte(lines[len(lines)-1]), &info)
122
123
124
125
126
127
128
	if err != nil {
		return info, err
	}

	return info, err
}

129
130
131
// SubList stores two  maps[language][sub-formats]
// Automatic Captions are the automatically generated youtube captions
// Subtitles are the proper Subtitles
132
133
134
135
136
type SubList struct {
	AutomaticCaptions map[string][]string
	Subtitles         map[string][]string
}

137
// NewSubList creates a new reference to a SubList with empty members.
138
139
140
141
142
143
144
func NewSubList() *SubList {
	return &SubList{
		AutomaticCaptions: make(map[string][]string),
		Subtitles:         make(map[string][]string),
	}
}

145
func listSubs(id string) (*SubList, error) {
146

147
	subs := NewSubList()
148

149
150
151
152
153
154
	args := []string{
		"--skip-download",
		"--list-subs",
		"https://youtu.be/" + id,
	}
	output, err := runYoutubeDL(args)
155
156
157
	if err != nil {
		return nil, err
	}
158
159
160

	lines := strings.Split(string(output), "\n")

161
	section := ""
162
	for _, line := range lines {
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
		if strings.Contains(strings.ToLower(line), "youtube") {
			continue
		}
		if strings.Contains(strings.ToLower(line), "available automatic captions") {
			section = "automatic captions"
			continue
		}
		if strings.Contains(strings.ToLower(line), "available subtitles") {
			section = "available subtitles"
			continue
		}
		if strings.Contains(strings.ToLower(line), "language formats") {
			continue
		}
		if section != "" {
			space := regexp.MustCompile(`\s+`)
			line := space.ReplaceAllString(line, " ")
			line = strings.ReplaceAll(line, ",", "")
			fields := strings.Split(string(line), " ")
			if len(fields) >= 2 {
				if section == "automatic captions" {
					subs.AutomaticCaptions[fields[0]] = fields[1:]

				}
				if section == "available subtitles" {
					subs.Subtitles[fields[0]] = fields[1:]
				}
			}
		}
192
193
	}

194
195
196
	return subs, nil
}

197
// findSub return nil when subtitle in the requested language is not available in any format
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
func findSub(id string, language string) (formats []string, err error) {
	subList, err := listSubs(id)
	if err != nil {
		return nil, err
	}
	if formats, ok := subList.Subtitles[language]; ok {
		return formats, nil
	}
	return nil, nil
}

//  findAutomaticCaption return nil when subtitle not available in any format
func findAutomaticCaption(id string, language string) (formats []string, err error) {
	subList, err := listSubs(id)
	if err != nil {
		return nil, err
	}
	if formats, ok := subList.AutomaticCaptions[language]; ok {
		return formats, nil
	}
	return nil, nil
}

// downloadSubtitle downloads the subtitle of the given youtube video in the given language and format and saves it as file to $file.$language.$format.
func downloadSubtitle(auto bool, id string, language string, format string, file string) error {
	subType := "--write-sub"
	if auto {
		subType = "--write-auto-sub"
	}

	args := []string{
		"--skip-download",
		subType,
		"--sub-lang",
		language,
		"--sub-format",
		format,
		"https://youtu.be/" + id,
		"-o",
		file,
	}

	output, err := runYoutubeDL(args)
	if err != nil {
		return err
	}
	if strings.Contains(strings.ToLower(string(output)), "subtitles not available") {
		return ErrYoutubeDL(fmt.Sprintf("subtitles language %s not available", language))
	}
	if strings.Contains(strings.ToLower(string(output)), "no subtitle format found") {
		return ErrYoutubeDL(fmt.Sprintf("subtitle format %s not found", format))

		//TODO delete the alternative subtitle file that it downloads automatically
	}

	return nil
}

// GetSubtitle downloads the subtitle of the video id in the passed language.
// Throws an error if subtitle not found.
258
// Downlaods the automatic caption when auto is true.
259
func GetSubtitle(auto bool, id string, language string) (sub annotation.Caption, err error) {
260

261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
	var formats []string
	if auto == false {
		formats, err = findSub(id, language)
		if err != nil {
			return nil, err
		}

		if formats == nil {
			return nil, ErrYoutubeDLLangNotSupported(fmt.Sprintf("No subtitle with language %s for video %s", language, id))
		}
	} else {
		formats, err = findAutomaticCaption(id, language)
		if err != nil {
			return nil, err
		}

		if formats == nil {
			return nil, ErrYoutubeDLLangNotSupported(fmt.Sprintf("No automatic caption with language %s for video %s", language, id))
		}
280
281
	}

282
283
284
285
286
287
	log.WithFields(log.Fields{
		"videoID":  id,
		"language": language,
		"auto":     auto,
		"types":    formats,
	}).Tracef("Available subtitle types for this video")
288

289
	for _, preferredSubType := range SupportedSubs {
290

291
		for _, format := range formats {
292

293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
			if preferredSubType.String() == format {

				filename := filepath.Join("/tmp", "sub-"+id)
				file := filename + "." + language + "." + format

				err := downloadSubtitle(auto, id, language, format, filename)
				defer os.Remove(file)
				if err != nil {
					log.Warn(err)
					continue
				}

				sub, err := astisub.OpenFile(file)
				if err != nil {
					log.Warn(err)
					continue
				}
				return &Subtitle{sub}, nil
			}
312
		}
313
314
315
316
317
318

		log.WithFields(log.Fields{"videoID": id,
			"language": language,
			"type":     preferredSubType.String(),
		}).Debugf("Preferred subtitle type not available for this video")

319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
	}

	return nil, ErrYoutubeDL("No available subtitle is supported")
}

// GetVideoURL retrieves the videoplayback url of the video and audio stream
func GetVideoURL(id string) (video *url.URL, audio *url.URL, err error) {

	args := []string{
		"-g",
		"https://www.youtube.com/watch?v=" + id,
	}

	// run command
	cmd := exec.Command("youtube-dl", args...)
334
	out, err := cmd.Output()
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
	if err != nil {
		return nil, nil, err
	}
	links := strings.Split(strings.Trim(string(out), "\n\t "), "\n")
	if len(links) != 2 {
		return nil, nil, ErrYoutubeDL(fmt.Sprintf("Expected two links but got %v", len(links)))
	}
	video, err = url.Parse(links[0])
	if err != nil {
		return nil, nil, err
	}
	audio, err = url.Parse(links[1])
	if err != nil {
		return nil, nil, err
	}
	return video, audio, nil
351
}
352

353
354
// GetPlaylistVideos returns a list of youtube video id's that are in the given playlist.
// The response is limited to the first 50 videos.
355
356
357
358
359
360
361
362
363
364
func GetPlaylistVideos(playListID string, apiKey string) (members []string, err error) {
	client := &http.Client{
		Transport: &transport.APIKey{Key: apiKey},
	}

	service, err := youtube.NewService(context.Background(), option.WithHTTPClient(client))
	if err != nil {
		return nil, err
	}
	// Make the API call to YouTube.
365
366
367
368
	call := service.PlaylistItems.List("id,contentDetails").
		PlaylistId(playListID).
		MaxResults(50)

369
370
371
372
373
374
375
376
377
378
379
	response, err := call.Do()
	if err != nil {
		return nil, err
	}

	for _, item := range response.Items {
		members = append(members, item.ContentDetails.VideoId)
	}

	return members, nil
}
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534

// LoadYouTubeAnnotationsVideo downloads the subtitles of the specified YouTube video and creates annotations in the specified mongo db collection
// user supplied YouTube subtitles are preferred over auto-generated ones
func LoadYouTubeAnnotationsVideo(videoID string, language string, mongoCollection string) error {
	var subs annotation.Caption
	err := errors.New("")

	// priorize manual subtitle before downloading automatic youtube caption (autogenerated)
	subs, err = GetSubtitle(false, videoID, language)
	if err != nil {
		_, ok := err.(ErrYoutubeDLLangNotSupported)
		if ok {
			log.WithFields(log.Fields{"videoID": videoID, "language": language}).Trace("No manual subtitle found")
			// download automatic caption
			subs, err = GetSubtitle(true, videoID, language)
			if err != nil {
				return err
			}

		}
	}

	annotations := subs.Annotations("youtube://" + videoID)

	log.WithFields(log.Fields{"videoID": videoID, "language": language, "quantity": len(annotations)}).Trace("Subtitle items found")

	mongoPort, err := strconv.Atoi(os.Getenv("MONGO_PORT"))
	collection, err := annotation.ConnectMongoDBCollection(os.Getenv("MONGO_HOST"), mongoPort, os.Getenv("MONGO_DB_NAME"), mongoCollection)
	if err != nil {
		return err
	}

	err = annotation.UploadToDB(annotations, collection)
	if err != nil {
		return err
	}

	return nil
}

// LoadYouTubeAnnotationsPlaylist downloads the subtitles of the specified YouTube playlist and loads them to the specified mongo db collection
// user supplied YouTube subtitles are preferred over auto-generated ones
func LoadYouTubeAnnotationsPlaylist(playlistID string, language string, mongoCollection string) error {
	defer monitor.Elapsed()()

	members, err := GetPlaylistVideos(playlistID, os.Getenv("YOUTUBE_DATA_API_KEY"))
	if err != nil {
		return err
	}
	log.WithFields(log.Fields{"playlistID": playlistID, "language": language, "quantity": len(members)}).Trace("Videos found in playlist")

	for _, videoID := range members {
		err := LoadYouTubeAnnotationsVideo(videoID, language, mongoCollection)
		if err != nil {
			log.Warning(err)
			continue
		}
	}

	return nil
}

// TimedText describes a parsed YouTube caption file type of .srv3 format (xml)
type TimedText struct {
	XMLName xml.Name `xml:"timedtext"`
	Format  string   `xml:"format,attr,omitempty"`
	Body    Body     `xml:"body"`
}

// Body contains all the paragraphs
type Body struct {
	Paragraphs []Paragraph `xml:"p"`
}

// Paragraph is one caption line (sequence of words=segments)
// Time [ms] is relative to the start of the caption (t=0)
// Duration ??
type Paragraph struct {
	Time     int       `xml:"t,attr"`
	Duration int       `xml:"d,attr"`
	Segments []Segment `xml:"s,omitempty"`
}

// Segment is one words/term with the Time [ms] (relative to the Paragraph.Time) and Duration [ms]
type Segment struct {
	Time     int    `xml:"t,attr,omitempty"`
	Duration int    `xml:"ac,attr,"`
	Value    string `xml:",chardata"`
}

// parseSRV decodes a youtubedl subtitle of type .srv3 into a TimedStruct
func parseSRV3(file []byte) (*TimedText, error) {
	var timedText TimedText

	err := xml.Unmarshal(file, &timedText)
	if err != nil {
		return nil, err
	}
	return &timedText, nil
}

// Annotations turns the TimedText into annotations with the src field set as given
// Method of the annotation.Caption IF
func (s *TimedText) Annotations(src string) []*annotation.Annotation {

	return fromTimedText(s, 3, src)
}

// fromTimedText applies the window slide approach to create annotations from TimedText
// window slide only applied within one paragraph
func fromTimedText(subtitle *TimedText, windowLength int, src string) []*annotation.Annotation {

	var windowAnnotations []*annotation.Annotation

	for paragraphNum, paragraph := range subtitle.Body.Paragraphs {

		for startSegmentIdx := range paragraph.Segments {

			for wLength := 1; wLength <= windowLength; wLength++ {

				// Detect if sliding window end exceeds segment end
				if startSegmentIdx+wLength > len(paragraph.Segments) {
					break
				}

				// Collect the values (text) of the current window
				var windowSegmentsValue []string
				for i := startSegmentIdx; i < startSegmentIdx+wLength; i++ {
					windowSegmentsValue = append(windowSegmentsValue, paragraph.Segments[i].Value)
				}

				// Compute the start time of the current window
				startSegment := paragraph.Segments[startSegmentIdx]
				endSegment := paragraph.Segments[startSegmentIdx+wLength-1]
				startTime := paragraph.Time + startSegment.Time
				endTime := paragraph.Time + endSegment.Time + endSegment.Duration

				// Create annotation of the current window
				a := &annotation.Annotation{
					Src: src,
					Subtitle: annotation.Subtitle{
						Count: paragraphNum,
						Text:  util.WordsToString(windowSegmentsValue),
						Start: startTime,
						End:   endTime,
					},
				}

				windowAnnotations = append(windowAnnotations, a)
			}
		}
	}

	return windowAnnotations
}