youtubedl.go 8.39 KB
Newer Older
1
package youtubedl
2
3

import (
4
	"context"
5
	"encoding/json"
6
7
8
	"fmt"
	"github.com/asticode/go-astisub"
	log "github.com/sirupsen/logrus"
9
10
11
12
	"google.golang.org/api/googleapi/transport"
	"google.golang.org/api/option"
	"google.golang.org/api/youtube/v3"
	"net/http"
13
14
	"net/url"
	"os"
15
	"os/exec"
16
17
	"path/filepath"
	"regexp"
18
19
20
	"strings"
)

21
// SubtitleFormat type describes a subtitle type (e.g. `vtt`)
22
23
24
25
26
27
28
type SubtitleFormat string

func (sf SubtitleFormat) String() string {
	return string(sf)
}

const (
29
30
31
32
33
34
35
36
37
	// SubVTT caption of type vtt
	SubVTT SubtitleFormat = "vtt"
	// SubSRT caption of type srt
	SubSRT SubtitleFormat = "srt"
	// SubSSA caption of type ssa
	SubSSA SubtitleFormat = "ssa"
	// SubSTL caption of type stl
	SubSTL SubtitleFormat = "stl"
	// SubTTML caption of type ttml
38
39
40
	SubTTML SubtitleFormat = "ttml"
)

41
// SupportedSubs contains all supported subtitle formats as key. Value is an empty string.
Christof Gerber's avatar
Christof Gerber committed
42
var SupportedSubs = map[SubtitleFormat]string{
43
	//SubVTT:  "", // TODO support vtt from YouTube with timestamp tags
Christof Gerber's avatar
Christof Gerber committed
44
45
46
47
	SubSRT:  "",
	SubSSA:  "",
	SubSTL:  "",
	SubTTML: "",
48
49
}

50
// ErrYoutubeDLLangNotSupported is thrown when language not supported
51
52
53
54
55
56
type ErrYoutubeDLLangNotSupported string

func (e ErrYoutubeDLLangNotSupported) Error() string {
	return string(e)
}

57
58
59
60
61
62
63
// ErrYoutubeVideoUnavailable is thrown when language not supported
type ErrYoutubeVideoUnavailable string

func (e ErrYoutubeVideoUnavailable) Error() string {
	return string(e)
}

64
// ErrYoutubeDL are errors with related to the youtubedl pkg
65
66
67
68
69
70
type ErrYoutubeDL string

func (e ErrYoutubeDL) Error() string {
	return string(e)
}

71
72
func runYoutubeDL(args []string) (output []byte, err error) {
	cmd := exec.Command("youtube-dl", args...)
73
	log.Debug(args)
74
	output, err = cmd.CombinedOutput()
75
	log.Trace(string(output))
76
77
	if err != nil && strings.Contains(string(output), "video is unavailable") {
		return nil, ErrYoutubeVideoUnavailable(string(output))
78
	}
79
	return output, err
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
}

func getVideoInfo(id string) (info map[string]interface{}, err error) {

	args := []string{
		"--skip-download",
		"-J",
		"https://youtu.be/" + id,
	}

	output, err := runYoutubeDL(args)
	if err != nil {
		return info, err
	}

95
96
97
98
99
100
101
	lines := strings.Split(strings.Trim(string(output), "\n\t "), "\n")

	if len(lines) == 0 {
		return nil, ErrYoutubeDL("youtube-dl response decoding error")
	}

	err = json.Unmarshal([]byte(lines[len(lines)-1]), &info)
102
103
104
105
106
107
108
	if err != nil {
		return info, err
	}

	return info, err
}

109
110
111
// SubList stores two  maps[language][sub-formats]
// Automatic Captions are the automatically generated youtube captions
// Subtitles are the proper Subtitles
112
113
114
115
116
type SubList struct {
	AutomaticCaptions map[string][]string
	Subtitles         map[string][]string
}

117
// NewSubList creates a new reference to a SubList with empty members.
118
119
120
121
122
123
124
func NewSubList() *SubList {
	return &SubList{
		AutomaticCaptions: make(map[string][]string),
		Subtitles:         make(map[string][]string),
	}
}

125
func listSubs(id string) (*SubList, error) {
126

127
	subs := NewSubList()
128

129
130
131
132
133
134
	args := []string{
		"--skip-download",
		"--list-subs",
		"https://youtu.be/" + id,
	}
	output, err := runYoutubeDL(args)
135
136
137
	if err != nil {
		return nil, err
	}
138
139
140

	lines := strings.Split(string(output), "\n")

141
	section := ""
142
	for _, line := range lines {
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
		if strings.Contains(strings.ToLower(line), "youtube") {
			continue
		}
		if strings.Contains(strings.ToLower(line), "available automatic captions") {
			section = "automatic captions"
			continue
		}
		if strings.Contains(strings.ToLower(line), "available subtitles") {
			section = "available subtitles"
			continue
		}
		if strings.Contains(strings.ToLower(line), "language formats") {
			continue
		}
		if section != "" {
			space := regexp.MustCompile(`\s+`)
			line := space.ReplaceAllString(line, " ")
			line = strings.ReplaceAll(line, ",", "")
			fields := strings.Split(string(line), " ")
			if len(fields) >= 2 {
				if section == "automatic captions" {
					subs.AutomaticCaptions[fields[0]] = fields[1:]

				}
				if section == "available subtitles" {
					subs.Subtitles[fields[0]] = fields[1:]
				}
			}
		}
172
173
	}

174
175
176
	return subs, nil
}

177
// findSub return nil when subtitle in the requested language is not available in any format
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
func findSub(id string, language string) (formats []string, err error) {
	subList, err := listSubs(id)
	if err != nil {
		return nil, err
	}
	if formats, ok := subList.Subtitles[language]; ok {
		return formats, nil
	}
	return nil, nil
}

//  findAutomaticCaption return nil when subtitle not available in any format
func findAutomaticCaption(id string, language string) (formats []string, err error) {
	subList, err := listSubs(id)
	if err != nil {
		return nil, err
	}
	if formats, ok := subList.AutomaticCaptions[language]; ok {
		return formats, nil
	}
	return nil, nil
}

// downloadSubtitle downloads the subtitle of the given youtube video in the given language and format and saves it as file to $file.$language.$format.
func downloadSubtitle(auto bool, id string, language string, format string, file string) error {
	subType := "--write-sub"
	if auto {
		subType = "--write-auto-sub"
	}

	args := []string{
		"--skip-download",
		subType,
		"--sub-lang",
		language,
		"--sub-format",
		format,
		"https://youtu.be/" + id,
		"-o",
		file,
	}

	output, err := runYoutubeDL(args)
	if err != nil {
		return err
	}
	if strings.Contains(strings.ToLower(string(output)), "subtitles not available") {
		return ErrYoutubeDL(fmt.Sprintf("subtitles language %s not available", language))
	}
	if strings.Contains(strings.ToLower(string(output)), "no subtitle format found") {
		return ErrYoutubeDL(fmt.Sprintf("subtitle format %s not found", format))

		//TODO delete the alternative subtitle file that it downloads automatically
	}

	return nil
}

// GetSubtitle downloads the subtitle of the video id in the passed language.
// Throws an error if subtitle not found.
238
// Downlaods the automatic caption when auto is true.
239
240
func GetSubtitle(auto bool, id string, language string) (sub *astisub.Subtitles, err error) {

241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
	var formats []string
	if auto == false {
		formats, err = findSub(id, language)
		if err != nil {
			return nil, err
		}

		if formats == nil {
			return nil, ErrYoutubeDLLangNotSupported(fmt.Sprintf("No subtitle with language %s for video %s", language, id))
		}
	} else {
		formats, err = findAutomaticCaption(id, language)
		if err != nil {
			return nil, err
		}

		if formats == nil {
			return nil, ErrYoutubeDLLangNotSupported(fmt.Sprintf("No automatic caption with language %s for video %s", language, id))
		}
260
261
262
263
264
265
266
267
268
	}

	for _, format := range formats {

		if _, ok := SupportedSubs[SubtitleFormat(format)]; !ok {
			log.Infof("Subtitle %s is not supported", format)
			continue
		}
		filename := filepath.Join("/tmp", "sub-"+id)
269
		file := filename + "." + language + "." + format
270

271
		err := downloadSubtitle(auto, id, language, format, filename)
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
		defer os.Remove(file)
		if err != nil {
			log.Warn(err)
			continue
		}

		sub, err := astisub.OpenFile(file)
		if err != nil {
			log.Warn(err)
			continue
		}
		return sub, nil
	}

	return nil, ErrYoutubeDL("No available subtitle is supported")
}

// GetVideoURL retrieves the videoplayback url of the video and audio stream
func GetVideoURL(id string) (video *url.URL, audio *url.URL, err error) {

	args := []string{
		"-g",
		"https://www.youtube.com/watch?v=" + id,
	}

	// run command
	cmd := exec.Command("youtube-dl", args...)
299
	out, err := cmd.Output()
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
	if err != nil {
		return nil, nil, err
	}
	links := strings.Split(strings.Trim(string(out), "\n\t "), "\n")
	if len(links) != 2 {
		return nil, nil, ErrYoutubeDL(fmt.Sprintf("Expected two links but got %v", len(links)))
	}
	video, err = url.Parse(links[0])
	if err != nil {
		return nil, nil, err
	}
	audio, err = url.Parse(links[1])
	if err != nil {
		return nil, nil, err
	}
	return video, audio, nil
316
}
317

318
319
// GetPlaylistVideos returns a list of youtube video id's that are in the given playlist.
// The response is limited to the first 50 videos.
320
321
322
323
324
325
326
327
328
329
func GetPlaylistVideos(playListID string, apiKey string) (members []string, err error) {
	client := &http.Client{
		Transport: &transport.APIKey{Key: apiKey},
	}

	service, err := youtube.NewService(context.Background(), option.WithHTTPClient(client))
	if err != nil {
		return nil, err
	}
	// Make the API call to YouTube.
330
331
332
333
	call := service.PlaylistItems.List("id,contentDetails").
		PlaylistId(playListID).
		MaxResults(50)

334
335
336
337
338
339
340
341
342
343
344
	response, err := call.Do()
	if err != nil {
		return nil, err
	}

	for _, item := range response.Items {
		members = append(members, item.ContentDetails.VideoId)
	}

	return members, nil
}