youtubedl.go 8.12 KB
Newer Older
1
package youtubedl
2
3

import (
4
	"context"
5
	"encoding/json"
6
7
8
	"fmt"
	"github.com/asticode/go-astisub"
	log "github.com/sirupsen/logrus"
9
10
11
12
	"google.golang.org/api/googleapi/transport"
	"google.golang.org/api/option"
	"google.golang.org/api/youtube/v3"
	"net/http"
13
14
	"net/url"
	"os"
15
	"os/exec"
16
17
	"path/filepath"
	"regexp"
18
19
20
	"strings"
)

21
// SubtitleFormat type describes a subtitle type (e.g. `vtt`)
22
23
24
25
26
27
28
type SubtitleFormat string

func (sf SubtitleFormat) String() string {
	return string(sf)
}

const (
29
30
31
32
33
34
35
36
37
	// SubVTT caption of type vtt
	SubVTT SubtitleFormat = "vtt"
	// SubSRT caption of type srt
	SubSRT SubtitleFormat = "srt"
	// SubSSA caption of type ssa
	SubSSA SubtitleFormat = "ssa"
	// SubSTL caption of type stl
	SubSTL SubtitleFormat = "stl"
	// SubTTML caption of type ttml
38
39
40
	SubTTML SubtitleFormat = "ttml"
)

41
// SupportedSubs contains all supported subtitle formats as key. Value is an empty string.
Christof Gerber's avatar
Christof Gerber committed
42
var SupportedSubs = map[SubtitleFormat]string{
43
	//SubVTT:  "", // TODO support vtt from YouTube with timestamp tags
Christof Gerber's avatar
Christof Gerber committed
44
45
46
47
	SubSRT:  "",
	SubSSA:  "",
	SubSTL:  "",
	SubTTML: "",
48
49
}

50
// ErrYoutubeDLLangNotSupported is thrown when language not supported
51
52
53
54
55
56
type ErrYoutubeDLLangNotSupported string

func (e ErrYoutubeDLLangNotSupported) Error() string {
	return string(e)
}

57
// ErrYoutubeDL are errors with related to the youtubedl pkg
58
59
60
61
62
63
type ErrYoutubeDL string

func (e ErrYoutubeDL) Error() string {
	return string(e)
}

64
65
func runYoutubeDL(args []string) (output []byte, err error) {
	cmd := exec.Command("youtube-dl", args...)
66
	log.Debug(args)
67
	output, err = cmd.CombinedOutput()
68
	log.Trace(string(output))
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
	if err != nil {
		return nil, err
	}

	return output, nil
}

func getVideoInfo(id string) (info map[string]interface{}, err error) {

	args := []string{
		"--skip-download",
		"-J",
		"https://youtu.be/" + id,
	}

	output, err := runYoutubeDL(args)
	if err != nil {
		return info, err
	}

89
90
91
92
93
94
95
	lines := strings.Split(strings.Trim(string(output), "\n\t "), "\n")

	if len(lines) == 0 {
		return nil, ErrYoutubeDL("youtube-dl response decoding error")
	}

	err = json.Unmarshal([]byte(lines[len(lines)-1]), &info)
96
97
98
99
100
101
102
	if err != nil {
		return info, err
	}

	return info, err
}

103
104
105
// SubList stores two  maps[language][sub-formats]
// Automatic Captions are the automatically generated youtube captions
// Subtitles are the proper Subtitles
106
107
108
109
110
type SubList struct {
	AutomaticCaptions map[string][]string
	Subtitles         map[string][]string
}

111
// NewSubList creates a new reference to a SubList with empty members.
112
113
114
115
116
117
118
119
120
121
122
func NewSubList() *SubList {
	return &SubList{
		AutomaticCaptions: make(map[string][]string),
		Subtitles:         make(map[string][]string),
	}
}

func listSubs(id string) (subs *SubList, err error) {

	subs = NewSubList()

123
124
125
126
127
128
	args := []string{
		"--skip-download",
		"--list-subs",
		"https://youtu.be/" + id,
	}
	output, err := runYoutubeDL(args)
129
130
131
	if err != nil {
		return nil, err
	}
132
133
134

	lines := strings.Split(string(output), "\n")

135
	section := ""
136
	for _, line := range lines {
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
		if strings.Contains(strings.ToLower(line), "youtube") {
			continue
		}
		if strings.Contains(strings.ToLower(line), "available automatic captions") {
			section = "automatic captions"
			continue
		}
		if strings.Contains(strings.ToLower(line), "available subtitles") {
			section = "available subtitles"
			continue
		}
		if strings.Contains(strings.ToLower(line), "language formats") {
			continue
		}
		if section != "" {
			space := regexp.MustCompile(`\s+`)
			line := space.ReplaceAllString(line, " ")
			line = strings.ReplaceAll(line, ",", "")
			fields := strings.Split(string(line), " ")
			if len(fields) >= 2 {
				if section == "automatic captions" {
					subs.AutomaticCaptions[fields[0]] = fields[1:]

				}
				if section == "available subtitles" {
					subs.Subtitles[fields[0]] = fields[1:]
				}
			}
		}
166
167
	}

168
169
170
	return subs, nil
}

171
// findSub return nil when subtitle in the requested language is not available in any format
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
func findSub(id string, language string) (formats []string, err error) {
	subList, err := listSubs(id)
	if err != nil {
		return nil, err
	}
	if formats, ok := subList.Subtitles[language]; ok {
		return formats, nil
	}
	return nil, nil
}

//  findAutomaticCaption return nil when subtitle not available in any format
func findAutomaticCaption(id string, language string) (formats []string, err error) {
	subList, err := listSubs(id)
	if err != nil {
		return nil, err
	}
	if formats, ok := subList.AutomaticCaptions[language]; ok {
		return formats, nil
	}
	return nil, nil
}

// downloadSubtitle downloads the subtitle of the given youtube video in the given language and format and saves it as file to $file.$language.$format.
func downloadSubtitle(auto bool, id string, language string, format string, file string) error {
	subType := "--write-sub"
	if auto {
		subType = "--write-auto-sub"
	}

	args := []string{
		"--skip-download",
		subType,
		"--sub-lang",
		language,
		"--sub-format",
		format,
		"https://youtu.be/" + id,
		"-o",
		file,
	}

	output, err := runYoutubeDL(args)
	if err != nil {
		return err
	}
	if strings.Contains(strings.ToLower(string(output)), "subtitles not available") {
		return ErrYoutubeDL(fmt.Sprintf("subtitles language %s not available", language))
	}
	if strings.Contains(strings.ToLower(string(output)), "no subtitle format found") {
		return ErrYoutubeDL(fmt.Sprintf("subtitle format %s not found", format))

		//TODO delete the alternative subtitle file that it downloads automatically
	}

	return nil
}

// GetSubtitle downloads the subtitle of the video id in the passed language.
// Throws an error if subtitle not found.
232
// Downlaods the automatic caption when auto is true.
233
234
func GetSubtitle(auto bool, id string, language string) (sub *astisub.Subtitles, err error) {

235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
	var formats []string
	if auto == false {
		formats, err = findSub(id, language)
		if err != nil {
			return nil, err
		}

		if formats == nil {
			return nil, ErrYoutubeDLLangNotSupported(fmt.Sprintf("No subtitle with language %s for video %s", language, id))
		}
	} else {
		formats, err = findAutomaticCaption(id, language)
		if err != nil {
			return nil, err
		}

		if formats == nil {
			return nil, ErrYoutubeDLLangNotSupported(fmt.Sprintf("No automatic caption with language %s for video %s", language, id))
		}
254
255
256
257
258
259
260
261
262
	}

	for _, format := range formats {

		if _, ok := SupportedSubs[SubtitleFormat(format)]; !ok {
			log.Infof("Subtitle %s is not supported", format)
			continue
		}
		filename := filepath.Join("/tmp", "sub-"+id)
263
		file := filename + "." + language + "." + format
264

265
		err := downloadSubtitle(auto, id, language, format, filename)
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
		defer os.Remove(file)
		if err != nil {
			log.Warn(err)
			continue
		}

		sub, err := astisub.OpenFile(file)
		if err != nil {
			log.Warn(err)
			continue
		}
		return sub, nil
	}

	return nil, ErrYoutubeDL("No available subtitle is supported")
}

// GetVideoURL retrieves the videoplayback url of the video and audio stream
func GetVideoURL(id string) (video *url.URL, audio *url.URL, err error) {

	args := []string{
		"-g",
		"https://www.youtube.com/watch?v=" + id,
	}

	// run command
	cmd := exec.Command("youtube-dl", args...)
293
	out, err := cmd.Output()
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
	if err != nil {
		return nil, nil, err
	}
	links := strings.Split(strings.Trim(string(out), "\n\t "), "\n")
	if len(links) != 2 {
		return nil, nil, ErrYoutubeDL(fmt.Sprintf("Expected two links but got %v", len(links)))
	}
	video, err = url.Parse(links[0])
	if err != nil {
		return nil, nil, err
	}
	audio, err = url.Parse(links[1])
	if err != nil {
		return nil, nil, err
	}
	return video, audio, nil
310
}
311

312
313
// GetPlaylistVideos returns a list of youtube video id's that are in the given playlist.
// The response is limited to the first 50 videos.
314
315
316
317
318
319
320
321
322
323
func GetPlaylistVideos(playListID string, apiKey string) (members []string, err error) {
	client := &http.Client{
		Transport: &transport.APIKey{Key: apiKey},
	}

	service, err := youtube.NewService(context.Background(), option.WithHTTPClient(client))
	if err != nil {
		return nil, err
	}
	// Make the API call to YouTube.
324
325
326
327
	call := service.PlaylistItems.List("id,contentDetails").
		PlaylistId(playListID).
		MaxResults(50)

328
329
330
331
332
333
334
335
336
337
338
	response, err := call.Do()
	if err != nil {
		return nil, err
	}

	for _, item := range response.Items {
		members = append(members, item.ContentDetails.VideoId)
	}

	return members, nil
}