Skip to content

GitLab

Commit c4ec3a01 authored by Christof Gerber's avatar Christof Gerber
Browse files
Options

Update to playlist support

parent 18bfb3a6
Pipeline #41831 failed with stages
      in 19 minutes and 48 seconds
      Hide whitespace changes
      Inline Side-by-side
      Showing with 39 additions and 17 deletions
      go.mod
      View file @ c4ec3a01
      ......@@ -7,9 +7,8 @@ require (
      github.com/sirupsen/logrus v1.4.2
      github.com/stretchr/testify v1.3.0
      gitlab.ethz.ch/chgerber/MessageComposition v0.0.0-20190606100759-1b591f45e7e5
      gitlab.ethz.ch/chgerber/annotation/v2 v2.0.0
      gitlab.ethz.ch/chgerber/annotation/v2 v2.2.0
      gitlab.ethz.ch/chgerber/monitor v0.0.0-20190527191251-2bb9dd731340
      go.mongodb.org/mongo-driver v1.0.3
      golang.org/x/tools v0.0.0-20190806215303-88ddfcebc769 // indirect
      google.golang.org/api v0.6.0
      )
      go.sum
      View file @ c4ec3a01
      ......@@ -21,10 +21,14 @@ github.com/asticode/go-astisub v0.0.0-20190514140258-c0ed7925c393 h1:3hJzbzuc1mY
      github.com/asticode/go-astisub v0.0.0-20190514140258-c0ed7925c393/go.mod h1:xeIVqWzWf4FmtftqxYgZccqq/ihzQifwXzNU8Wp0PUI=
      github.com/asticode/go-astitools v1.0.0 h1:B5l3sI+hgQJsUNyLgd6pHrhH75GAtFzZVIirso7oFeg=
      github.com/asticode/go-astitools v1.0.0/go.mod h1:E7f1P0KkBNgafRCD0dHqn41jNHyYHjxrnK1jH2s4pmA=
      github.com/asticode/go-astitools v1.1.0 h1:h5hVWUKB9eUZY7/mgu6Ic6Rke8ZiWHkiJO0rIGIb5j4=
      github.com/asticode/go-astitools v1.1.0/go.mod h1:EfgrhUJK6nM17TckqASIqR2W71uNUAoIonm6mNhUtaQ=
      github.com/asticode/go-astits v0.0.0-20190105100228-f4a041fc41e5 h1:0HrvsgiaVvsk15xUTOBxvetURkvi70N3PdsOk4cePOc=
      github.com/asticode/go-astits v0.0.0-20190105100228-f4a041fc41e5/go.mod h1:IO8rus+3gLhbOSb9nI40h7UIG5vh7T1LsmvyBvXZT20=
      github.com/asticode/go-astits v0.0.0-20190509074025-f5a59e81bd78 h1:Q9lWjAXSxnjrfYtsQRc1l4O5yBvu4N8rhZhtiIob5Hw=
      github.com/asticode/go-astits v0.0.0-20190509074025-f5a59e81bd78/go.mod h1:YjTpnbhUpZb5Ygl/mNpH6F0fpq1FKtGIB/tn9yFKvr8=
      github.com/asticode/go-astits v1.0.0 h1:RmuQnNVFKaiHWNKSXA6lqiVsgJXEGnof1WDqPzDqtXI=
      github.com/asticode/go-astits v1.0.0/go.mod h1:YjTpnbhUpZb5Ygl/mNpH6F0fpq1FKtGIB/tn9yFKvr8=
      github.com/aws/aws-sdk-go v1.19.5/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
      github.com/aws/aws-sdk-go v1.19.27/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
      github.com/aws/aws-sdk-go v1.19.43/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
      ......@@ -65,6 +69,7 @@ github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
      github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
      github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
      github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
      github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
      github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
      github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
      github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
      ......@@ -157,6 +162,8 @@ gitlab.ethz.ch/chgerber/annotation v0.0.0-20190605204157-8c12804d664b h1:bpfv2Zy
      gitlab.ethz.ch/chgerber/annotation v0.0.0-20190605204157-8c12804d664b/go.mod h1:uO/B1tPARzCcEcMc0+tMIs8S25SEbDaEDR2WzK6uUgU=
      gitlab.ethz.ch/chgerber/annotation/v2 v2.0.0 h1:04E+T0VHCCAHCsrGj973vocyUCx6I35Maodr70Z+tlk=
      gitlab.ethz.ch/chgerber/annotation/v2 v2.0.0/go.mod h1:LdyyrAsy8kSnM84LMqdtVDZB+mpTeA7ugE52jMer0Ww=
      gitlab.ethz.ch/chgerber/annotation/v2 v2.2.0 h1:zuFq2ci6ilDsT94pyzezxMuxUQJ6EU3EUBoZK2QTR+w=
      gitlab.ethz.ch/chgerber/annotation/v2 v2.2.0/go.mod h1:tU81QXYhR5sf9cUE/y14MMOEVGNXfNvYN6aOz0LyVdQ=
      gitlab.ethz.ch/chgerber/monitor v0.0.0-20190412120244-dcc5be809fae/go.mod h1:HqYam5C8lDlOiYTcNgbvi11jILmk1puFjAvVoehT36o=
      gitlab.ethz.ch/chgerber/monitor v0.0.0-20190527191251-2bb9dd731340 h1:8r29nRz4ldWxchnPY6DcJnivPDo5eXmvhNC2kdW8vpg=
      gitlab.ethz.ch/chgerber/monitor v0.0.0-20190527191251-2bb9dd731340/go.mod h1:pJs/THBAEx9qNq3xQdrGq+wcnQRxpNSpAj4dQH1cRMk=
      ......@@ -167,6 +174,7 @@ gitlab.ethz.ch/chgerber/s3helper v0.0.0-20190527212420-8196b9b9a04f/go.mod h1:DI
      gitlab.ethz.ch/chgerber/video v0.0.0-20190527150454-f0fa080cc6fa/go.mod h1:kx8UkJw8aKdcN3qf4FjrRc92L9ea4mPaUcopFbLZ4+A=
      gitlab.ethz.ch/chgerber/youtubedl v0.0.0-20190511081243-7baf0e636a58/go.mod h1:Z7ZPw8AdgFL1b3l1Ags1RDh3DdEoBLrf5rsO1BcxS+Q=
      gitlab.ethz.ch/chgerber/youtubedl v0.0.0-20190529074845-7f6c34a0a653/go.mod h1:cdyW37Y3w6nCfB1oRkfKOSYfxtuKd2GNCOrSz2GgtuQ=
      gitlab.ethz.ch/chgerber/youtubedl v1.0.0/go.mod h1:VqzDlLmKm5UnA7/ZxtFL8MnBs2a5YNz9lP/CYF7Tevk=
      go.mongodb.org/mongo-driver v1.0.2 h1:RwjK1tKt7VPqQh3tsjiEqKJg75GNhP/loch+PwRc4ig=
      go.mongodb.org/mongo-driver v1.0.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
      go.mongodb.org/mongo-driver v1.0.3 h1:GKoji1ld3tw2aC+GX1wbr/J2fX13yNacEYoJ8Nhr0yU=
      ......@@ -190,6 +198,10 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmV
      golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
      golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
      golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
      golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 h1:Gv7RPwsi3eZ2Fgewe3CBsuOebPwO27PoXzRpJPsvSSM=
      golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
      golang.org/x/crypto v0.0.0-20190909091759-094676da4a83 h1:mgAKeshyNqWKdENOnQsg+8dRTwZFIwFaO3HNl52sweA=
      golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
      golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
      golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
      golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
      ......@@ -225,6 +237,10 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR
      golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
      golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
      golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
      golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM=
      golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
      golang.org/x/net v0.0.0-20190909003024-a7b16738d86b h1:XfVGCX+0T4WOStkaOsJRllbsiImhB2jgVBGc9L0lPGc=
      golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
      golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
      golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=
      golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
      ......@@ -262,6 +278,10 @@ golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed h1:uPxWBzB3+mlnjy9W58qY1j/cj
      golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
      golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa h1:KIDDMLT1O0Nr7TSxp8xM5tJcdn8tgyAONntO829og1M=
      golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
      golang.org/x/sys v0.0.0-20190902133755-9109b7679e13 h1:tdsQdquKbTNMsSZLqnLELJGzCANp9oXhu6zFBW6ODx4=
      golang.org/x/sys v0.0.0-20190902133755-9109b7679e13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
      golang.org/x/sys v0.0.0-20190909082730-f460065e899a h1:mIzbOulag9/gXacgxKlFVwpCOWSfBT3/pDyyCwGA9as=
      golang.org/x/sys v0.0.0-20190909082730-f460065e899a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
      golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
      golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
      golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
      ......@@ -285,6 +305,7 @@ golang.org/x/tools v0.0.0-20190530171427-2b03ca6e44eb/go.mod h1:/rFqwRUd4F7ZHNgw
      golang.org/x/tools v0.0.0-20190603231351-8aaa1484dc10/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
      golang.org/x/tools v0.0.0-20190806215303-88ddfcebc769 h1:D/+0wZ7qKh5vQqpbxJGPnaMv1tuCCKmn6heUpPt3FOk=
      golang.org/x/tools v0.0.0-20190806215303-88ddfcebc769/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
      golang.org/x/tools v0.0.0-20190909030654-5b82db07426d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
      golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
      google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
      google.golang.org/api v0.3.2 h1:iTp+3yyl/KOtxa/d1/JUE0GGSoR6FuW5udver22iwpw=
      ......
      youtubedl.go
      View file @ c4ec3a01
      ......@@ -32,8 +32,8 @@ type Subtitle struct {
      // Annotations turns the subtitles into annotations with the src field set as given
      // Method of the annotation.Caption IF
      func (s *Subtitle) Annotations(src string) []*annotation.Annotation {
      return annotation.SubToAnnotation(s.Subtitles, src)
      func (s *Subtitle) Annotations(src string, playlist annotation.Playlist) []*annotation.Annotation {
      return annotation.SubToAnnotation(s.Subtitles, src, playlist)
      }
      // SubtitleFormat type describes a subtitle type (e.g. `vtt`)
      ......@@ -382,9 +382,9 @@ func GetPlaylistVideos(playListID string, apiKey string) (members []string, err
      // LoadYouTubeVideos downloads the subtitles of the specified YouTube videos and creates annotations in the specified mongo db collection
      // user supplied YouTube subtitles are preferred over auto-generated ones
      func LoadYouTubeVideos(videoIDs []string, language string, collection *mongo.Collection) error {
      func LoadYouTubeVideos(videoIDs []string, language string, collection *mongo.Collection, playlist annotation.Playlist) error {
      for _, v := range videoIDs {
      err := loadYouTubeAnnotationsVideo(v, language, collection)
      err := loadYouTubeAnnotationsVideo(v, language, collection, playlist)
      if err != nil {
      log.Warn(err)
      continue
      ......@@ -393,7 +393,7 @@ func LoadYouTubeVideos(videoIDs []string, language string, collection *mongo.Col
      return nil
      }
      func loadYouTubeAnnotationsVideo(videoID string, language string, collection *mongo.Collection) error {
      func loadYouTubeAnnotationsVideo(videoID string, language string, collection *mongo.Collection, playlist annotation.Playlist) error {
      var subs annotation.Caption
      err := errors.New("")
      ......@@ -414,7 +414,7 @@ func loadYouTubeAnnotationsVideo(videoID string, language string, collection *mo
      }
      }
      annotations := subs.Annotations("youtube://" + videoID)
      annotations := subs.Annotations("youtube://"+videoID, playlist)
      log.WithFields(log.Fields{"videoID": videoID, "language": language, "quantity": len(annotations)}).Trace("Subtitle items found")
      ......@@ -428,7 +428,7 @@ func loadYouTubeAnnotationsVideo(videoID string, language string, collection *mo
      // LoadYouTubeAnnotationsPlaylist downloads the subtitles of the specified YouTube playlist and loads them to the specified mongo db collection
      // user supplied YouTube subtitles are preferred over auto-generated ones
      func LoadYouTubeAnnotationsPlaylist(playlistID string, language string, collection *mongo.Collection) error {
      func LoadYouTubeAnnotationsPlaylist(playlistID string, language string, collection *mongo.Collection, playlist annotation.Playlist) error {
      defer monitor.Elapsed()()
      members, err := GetPlaylistVideos(playlistID, os.Getenv("YOUTUBE_DATA_API_KEY"))
      ......@@ -438,7 +438,7 @@ func LoadYouTubeAnnotationsPlaylist(playlistID string, language string, collecti
      log.WithFields(log.Fields{"playlistID": playlistID, "language": language, "quantity": len(members)}).Trace("Videos found in playlist")
      for _, videoID := range members {
      err := loadYouTubeAnnotationsVideo(videoID, language, collection)
      err := loadYouTubeAnnotationsVideo(videoID, language, collection, playlist)
      if err != nil {
      log.Warning(err)
      continue
      ......@@ -502,14 +502,14 @@ func parseSRV3(file []byte) (*TimedText, error) {
      // Args default values
      // 1. minWindowLength := 1
      // 2. maxWindowLength := 5
      func (s *TimedText) Annotations(src string) []*annotation.Annotation {
      func (s *TimedText) Annotations(src string, playlist annotation.Playlist) []*annotation.Annotation {
      return fromTimedText(s, src)
      return fromTimedText(s, src, playlist)
      }
      // fromTimedText creates one Annotation for each Paragraph
      // TODO find smart way which segments to combine into one annotation (e.g. when t_pause > x || len(segments) > 10)
      func fromTimedText(subtitle *TimedText, src string) []*annotation.Annotation {
      func fromTimedText(subtitle *TimedText, src string, playlist annotation.Playlist) []*annotation.Annotation {
      var annos []*annotation.Annotation
      ......@@ -521,7 +521,7 @@ func fromTimedText(subtitle *TimedText, src string) []*annotation.Annotation {
      count++
      // Create annotation for the paragraph
      a := annotation.Annotation{Src: src, Subtitle: annotation.Subtitle{Count: count}}
      a := annotation.Annotation{Src: src, Subtitle: annotation.Subtitle{Count: count}, Playlist: playlist}
      // handle when current paragraph subtitle stays displayed even when next one appears
      var endTime int
      ......
      youtubedl_test.go
      View file @ c4ec3a01
      ......@@ -130,7 +130,7 @@ func TestAutomaticCaptionDownload(t *testing.T) {
      func TestGetSubtitle(t *testing.T) {
      sub, err := GetSubtitle(false, "OsFEV35tWsg", "en")
      assert.Equal(t, nil, err)
      assert.Equal(t, "Philosophers, dramatists, theologians", sub.Annotations("")[0].Subtitle.Text)
      assert.Equal(t, "Philosophers, dramatists, theologians", sub.Annotations("", annotation.Playlist{ID: "1", Name: "TED: The psychology of evil"})[0].Subtitle.Text)
      }
      func TestUnavailableVideo(t *testing.T) {
      ......@@ -173,7 +173,7 @@ func TestTimedTextToAnnotationAutomaticCaption(t *testing.T) {
      timedText, err := parseSRV3(byteValue)
      assert.Equal(t, nil, err)
      a := timedText.Annotations("youtube://VIlLpnJJl_4")
      a := timedText.Annotations("youtube://VIlLpnJJl_4", annotation.Playlist{ID: "VIlLpnJJl_4", Name: "Ninja: Stories Of A Stream Sniper!"})
      aExpect := annotation.Annotation{
      Src: "youtube://VIlLpnJJl_4",
      ......@@ -193,6 +193,7 @@ func TestTimedTextToAnnotationAutomaticCaption(t *testing.T) {
      {Start: 3720, End: 4539, Text: "differently"},
      },
      },
      Playlist: annotation.Playlist{ID: "VIlLpnJJl_4", Name: "Ninja: Stories Of A Stream Sniper!"},
      }
      assert.Equal(t, aExpect, *a[0])
      ......@@ -212,6 +213,7 @@ func TestTimedTextToAnnotationAutomaticCaption(t *testing.T) {
      {Start: 60690, End: 60889, Text: "of"},
      },
      },
      Playlist: annotation.Playlist{ID: "VIlLpnJJl_4", Name: "Ninja: Stories Of A Stream Sniper!"},
      }
      assert.Equal(t, aExpect, *a[15])
      }
      ......@@ -267,7 +269,7 @@ func TestTimedTextToAnnotations(t *testing.T) {
      assert.Equal(t, nil, err)
      // Test for the right number of annotations
      annotations := fromTimedText(timedText, "")
      annotations := fromTimedText(timedText, "", annotation.Playlist{})
      assert.Equal(t, 8, len(annotations))
      assert.Equal(t, 8, len(annotations[0].Subtitle.Segments))
      assert.Equal(t, 9, len(annotations[1].Subtitle.Segments))
      ......
      Markdown is supported
      0% or .
      You are about to add 0 people to the discussion. Proceed with caution.
      Finish editing this message first!
      Please register or to comment