Skip to content

GitLab

Commit c4ec3a01 authored by Christof Gerber's avatar Christof Gerber
Browse files

Update to playlist support

parent 18bfb3a6
Pipeline #41831 failed with stages
in 19 minutes and 48 seconds
Hide whitespace changes
Inline Side-by-side
Showing with 39 additions and 17 deletions
go.mod
View file @ c4ec3a01
......@@ -7,9 +7,8 @@ require (
github.com/sirupsen/logrus v1.4.2
github.com/stretchr/testify v1.3.0
gitlab.ethz.ch/chgerber/MessageComposition v0.0.0-20190606100759-1b591f45e7e5
gitlab.ethz.ch/chgerber/annotation/v2 v2.0.0
gitlab.ethz.ch/chgerber/annotation/v2 v2.2.0
gitlab.ethz.ch/chgerber/monitor v0.0.0-20190527191251-2bb9dd731340
go.mongodb.org/mongo-driver v1.0.3
golang.org/x/tools v0.0.0-20190806215303-88ddfcebc769 // indirect
google.golang.org/api v0.6.0
)
go.sum
View file @ c4ec3a01
......@@ -21,10 +21,14 @@ github.com/asticode/go-astisub v0.0.0-20190514140258-c0ed7925c393 h1:3hJzbzuc1mY
github.com/asticode/go-astisub v0.0.0-20190514140258-c0ed7925c393/go.mod h1:xeIVqWzWf4FmtftqxYgZccqq/ihzQifwXzNU8Wp0PUI=
github.com/asticode/go-astitools v1.0.0 h1:B5l3sI+hgQJsUNyLgd6pHrhH75GAtFzZVIirso7oFeg=
github.com/asticode/go-astitools v1.0.0/go.mod h1:E7f1P0KkBNgafRCD0dHqn41jNHyYHjxrnK1jH2s4pmA=
github.com/asticode/go-astitools v1.1.0 h1:h5hVWUKB9eUZY7/mgu6Ic6Rke8ZiWHkiJO0rIGIb5j4=
github.com/asticode/go-astitools v1.1.0/go.mod h1:EfgrhUJK6nM17TckqASIqR2W71uNUAoIonm6mNhUtaQ=
github.com/asticode/go-astits v0.0.0-20190105100228-f4a041fc41e5 h1:0HrvsgiaVvsk15xUTOBxvetURkvi70N3PdsOk4cePOc=
github.com/asticode/go-astits v0.0.0-20190105100228-f4a041fc41e5/go.mod h1:IO8rus+3gLhbOSb9nI40h7UIG5vh7T1LsmvyBvXZT20=
github.com/asticode/go-astits v0.0.0-20190509074025-f5a59e81bd78 h1:Q9lWjAXSxnjrfYtsQRc1l4O5yBvu4N8rhZhtiIob5Hw=
github.com/asticode/go-astits v0.0.0-20190509074025-f5a59e81bd78/go.mod h1:YjTpnbhUpZb5Ygl/mNpH6F0fpq1FKtGIB/tn9yFKvr8=
github.com/asticode/go-astits v1.0.0 h1:RmuQnNVFKaiHWNKSXA6lqiVsgJXEGnof1WDqPzDqtXI=
github.com/asticode/go-astits v1.0.0/go.mod h1:YjTpnbhUpZb5Ygl/mNpH6F0fpq1FKtGIB/tn9yFKvr8=
github.com/aws/aws-sdk-go v1.19.5/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.19.27/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.19.43/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
......@@ -65,6 +69,7 @@ github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
......@@ -157,6 +162,8 @@ gitlab.ethz.ch/chgerber/annotation v0.0.0-20190605204157-8c12804d664b h1:bpfv2Zy
gitlab.ethz.ch/chgerber/annotation v0.0.0-20190605204157-8c12804d664b/go.mod h1:uO/B1tPARzCcEcMc0+tMIs8S25SEbDaEDR2WzK6uUgU=
gitlab.ethz.ch/chgerber/annotation/v2 v2.0.0 h1:04E+T0VHCCAHCsrGj973vocyUCx6I35Maodr70Z+tlk=
gitlab.ethz.ch/chgerber/annotation/v2 v2.0.0/go.mod h1:LdyyrAsy8kSnM84LMqdtVDZB+mpTeA7ugE52jMer0Ww=
gitlab.ethz.ch/chgerber/annotation/v2 v2.2.0 h1:zuFq2ci6ilDsT94pyzezxMuxUQJ6EU3EUBoZK2QTR+w=
gitlab.ethz.ch/chgerber/annotation/v2 v2.2.0/go.mod h1:tU81QXYhR5sf9cUE/y14MMOEVGNXfNvYN6aOz0LyVdQ=
gitlab.ethz.ch/chgerber/monitor v0.0.0-20190412120244-dcc5be809fae/go.mod h1:HqYam5C8lDlOiYTcNgbvi11jILmk1puFjAvVoehT36o=
gitlab.ethz.ch/chgerber/monitor v0.0.0-20190527191251-2bb9dd731340 h1:8r29nRz4ldWxchnPY6DcJnivPDo5eXmvhNC2kdW8vpg=
gitlab.ethz.ch/chgerber/monitor v0.0.0-20190527191251-2bb9dd731340/go.mod h1:pJs/THBAEx9qNq3xQdrGq+wcnQRxpNSpAj4dQH1cRMk=
......@@ -167,6 +174,7 @@ gitlab.ethz.ch/chgerber/s3helper v0.0.0-20190527212420-8196b9b9a04f/go.mod h1:DI
gitlab.ethz.ch/chgerber/video v0.0.0-20190527150454-f0fa080cc6fa/go.mod h1:kx8UkJw8aKdcN3qf4FjrRc92L9ea4mPaUcopFbLZ4+A=
gitlab.ethz.ch/chgerber/youtubedl v0.0.0-20190511081243-7baf0e636a58/go.mod h1:Z7ZPw8AdgFL1b3l1Ags1RDh3DdEoBLrf5rsO1BcxS+Q=
gitlab.ethz.ch/chgerber/youtubedl v0.0.0-20190529074845-7f6c34a0a653/go.mod h1:cdyW37Y3w6nCfB1oRkfKOSYfxtuKd2GNCOrSz2GgtuQ=
gitlab.ethz.ch/chgerber/youtubedl v1.0.0/go.mod h1:VqzDlLmKm5UnA7/ZxtFL8MnBs2a5YNz9lP/CYF7Tevk=
go.mongodb.org/mongo-driver v1.0.2 h1:RwjK1tKt7VPqQh3tsjiEqKJg75GNhP/loch+PwRc4ig=
go.mongodb.org/mongo-driver v1.0.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.mongodb.org/mongo-driver v1.0.3 h1:GKoji1ld3tw2aC+GX1wbr/J2fX13yNacEYoJ8Nhr0yU=
......@@ -190,6 +198,10 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmV
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 h1:Gv7RPwsi3eZ2Fgewe3CBsuOebPwO27PoXzRpJPsvSSM=
golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190909091759-094676da4a83 h1:mgAKeshyNqWKdENOnQsg+8dRTwZFIwFaO3HNl52sweA=
golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
......@@ -225,6 +237,10 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190909003024-a7b16738d86b h1:XfVGCX+0T4WOStkaOsJRllbsiImhB2jgVBGc9L0lPGc=
golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
......@@ -262,6 +278,10 @@ golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed h1:uPxWBzB3+mlnjy9W58qY1j/cj
golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa h1:KIDDMLT1O0Nr7TSxp8xM5tJcdn8tgyAONntO829og1M=
golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190902133755-9109b7679e13 h1:tdsQdquKbTNMsSZLqnLELJGzCANp9oXhu6zFBW6ODx4=
golang.org/x/sys v0.0.0-20190902133755-9109b7679e13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190909082730-f460065e899a h1:mIzbOulag9/gXacgxKlFVwpCOWSfBT3/pDyyCwGA9as=
golang.org/x/sys v0.0.0-20190909082730-f460065e899a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
......@@ -285,6 +305,7 @@ golang.org/x/tools v0.0.0-20190530171427-2b03ca6e44eb/go.mod h1:/rFqwRUd4F7ZHNgw
golang.org/x/tools v0.0.0-20190603231351-8aaa1484dc10/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190806215303-88ddfcebc769 h1:D/+0wZ7qKh5vQqpbxJGPnaMv1tuCCKmn6heUpPt3FOk=
golang.org/x/tools v0.0.0-20190806215303-88ddfcebc769/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190909030654-5b82db07426d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
google.golang.org/api v0.3.2 h1:iTp+3yyl/KOtxa/d1/JUE0GGSoR6FuW5udver22iwpw=
......
youtubedl.go
View file @ c4ec3a01
......@@ -32,8 +32,8 @@ type Subtitle struct {
// Annotations turns the subtitles into annotations with the src field set as given
// Method of the annotation.Caption IF
func (s *Subtitle) Annotations(src string) []*annotation.Annotation {
return annotation.SubToAnnotation(s.Subtitles, src)
func (s *Subtitle) Annotations(src string, playlist annotation.Playlist) []*annotation.Annotation {
return annotation.SubToAnnotation(s.Subtitles, src, playlist)
}
// SubtitleFormat type describes a subtitle type (e.g. `vtt`)
......@@ -382,9 +382,9 @@ func GetPlaylistVideos(playListID string, apiKey string) (members []string, err
// LoadYouTubeVideos downloads the subtitles of the specified YouTube videos and creates annotations in the specified mongo db collection
// user supplied YouTube subtitles are preferred over auto-generated ones
func LoadYouTubeVideos(videoIDs []string, language string, collection *mongo.Collection) error {
func LoadYouTubeVideos(videoIDs []string, language string, collection *mongo.Collection, playlist annotation.Playlist) error {
for _, v := range videoIDs {
err := loadYouTubeAnnotationsVideo(v, language, collection)
err := loadYouTubeAnnotationsVideo(v, language, collection, playlist)
if err != nil {
log.Warn(err)
continue
......@@ -393,7 +393,7 @@ func LoadYouTubeVideos(videoIDs []string, language string, collection *mongo.Col
return nil
}
func loadYouTubeAnnotationsVideo(videoID string, language string, collection *mongo.Collection) error {
func loadYouTubeAnnotationsVideo(videoID string, language string, collection *mongo.Collection, playlist annotation.Playlist) error {
var subs annotation.Caption
err := errors.New("")
......@@ -414,7 +414,7 @@ func loadYouTubeAnnotationsVideo(videoID string, language string, collection *mo
}
}
annotations := subs.Annotations("youtube://" + videoID)
annotations := subs.Annotations("youtube://"+videoID, playlist)
log.WithFields(log.Fields{"videoID": videoID, "language": language, "quantity": len(annotations)}).Trace("Subtitle items found")
......@@ -428,7 +428,7 @@ func loadYouTubeAnnotationsVideo(videoID string, language string, collection *mo
// LoadYouTubeAnnotationsPlaylist downloads the subtitles of the specified YouTube playlist and loads them to the specified mongo db collection
// user supplied YouTube subtitles are preferred over auto-generated ones
func LoadYouTubeAnnotationsPlaylist(playlistID string, language string, collection *mongo.Collection) error {
func LoadYouTubeAnnotationsPlaylist(playlistID string, language string, collection *mongo.Collection, playlist annotation.Playlist) error {
defer monitor.Elapsed()()
members, err := GetPlaylistVideos(playlistID, os.Getenv("YOUTUBE_DATA_API_KEY"))
......@@ -438,7 +438,7 @@ func LoadYouTubeAnnotationsPlaylist(playlistID string, language string, collecti
log.WithFields(log.Fields{"playlistID": playlistID, "language": language, "quantity": len(members)}).Trace("Videos found in playlist")
for _, videoID := range members {
err := loadYouTubeAnnotationsVideo(videoID, language, collection)
err := loadYouTubeAnnotationsVideo(videoID, language, collection, playlist)
if err != nil {
log.Warning(err)
continue
......@@ -502,14 +502,14 @@ func parseSRV3(file []byte) (*TimedText, error) {
// Args default values
// 1. minWindowLength := 1
// 2. maxWindowLength := 5
func (s *TimedText) Annotations(src string) []*annotation.Annotation {
func (s *TimedText) Annotations(src string, playlist annotation.Playlist) []*annotation.Annotation {
return fromTimedText(s, src)
return fromTimedText(s, src, playlist)
}
// fromTimedText creates one Annotation for each Paragraph
// TODO find smart way which segments to combine into one annotation (e.g. when t_pause > x || len(segments) > 10)
func fromTimedText(subtitle *TimedText, src string) []*annotation.Annotation {
func fromTimedText(subtitle *TimedText, src string, playlist annotation.Playlist) []*annotation.Annotation {
var annos []*annotation.Annotation
......@@ -521,7 +521,7 @@ func fromTimedText(subtitle *TimedText, src string) []*annotation.Annotation {
count++
// Create annotation for the paragraph
a := annotation.Annotation{Src: src, Subtitle: annotation.Subtitle{Count: count}}
a := annotation.Annotation{Src: src, Subtitle: annotation.Subtitle{Count: count}, Playlist: playlist}
// handle when current paragraph subtitle stays displayed even when next one appears
var endTime int
......
youtubedl_test.go
View file @ c4ec3a01
......@@ -130,7 +130,7 @@ func TestAutomaticCaptionDownload(t *testing.T) {
func TestGetSubtitle(t *testing.T) {
sub, err := GetSubtitle(false, "OsFEV35tWsg", "en")
assert.Equal(t, nil, err)
assert.Equal(t, "Philosophers, dramatists, theologians", sub.Annotations("")[0].Subtitle.Text)
assert.Equal(t, "Philosophers, dramatists, theologians", sub.Annotations("", annotation.Playlist{ID: "1", Name: "TED: The psychology of evil"})[0].Subtitle.Text)
}
func TestUnavailableVideo(t *testing.T) {
......@@ -173,7 +173,7 @@ func TestTimedTextToAnnotationAutomaticCaption(t *testing.T) {
timedText, err := parseSRV3(byteValue)
assert.Equal(t, nil, err)
a := timedText.Annotations("youtube://VIlLpnJJl_4")
a := timedText.Annotations("youtube://VIlLpnJJl_4", annotation.Playlist{ID: "VIlLpnJJl_4", Name: "Ninja: Stories Of A Stream Sniper!"})
aExpect := annotation.Annotation{
Src: "youtube://VIlLpnJJl_4",
......@@ -193,6 +193,7 @@ func TestTimedTextToAnnotationAutomaticCaption(t *testing.T) {
{Start: 3720, End: 4539, Text: "differently"},
},
},
Playlist: annotation.Playlist{ID: "VIlLpnJJl_4", Name: "Ninja: Stories Of A Stream Sniper!"},
}
assert.Equal(t, aExpect, *a[0])
......@@ -212,6 +213,7 @@ func TestTimedTextToAnnotationAutomaticCaption(t *testing.T) {
{Start: 60690, End: 60889, Text: "of"},
},
},
Playlist: annotation.Playlist{ID: "VIlLpnJJl_4", Name: "Ninja: Stories Of A Stream Sniper!"},
}
assert.Equal(t, aExpect, *a[15])
}
......@@ -267,7 +269,7 @@ func TestTimedTextToAnnotations(t *testing.T) {
assert.Equal(t, nil, err)
// Test for the right number of annotations
annotations := fromTimedText(timedText, "")
annotations := fromTimedText(timedText, "", annotation.Playlist{})
assert.Equal(t, 8, len(annotations))
assert.Equal(t, 8, len(annotations[0].Subtitle.Segments))
assert.Equal(t, 9, len(annotations[1].Subtitle.Segments))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment