Make loading multiple videos less verbose. endTime from next segment/paragraph

go.mod

@@ -11,8 +11,9 @@ require (
 	gitlab.ethz.ch/chgerber/annotation v0.0.0-20190607145645-0049de6d439e
 	gitlab.ethz.ch/chgerber/monitor v0.0.0-20190527191251-2bb9dd731340
 	go.mongodb.org/mongo-driver v1.0.3
-	golang.org/x/net v0.0.0-20190606173856-1492cefac77f // indirect
-	golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444 // indirect
+	golang.org/x/net v0.0.0-20190607181551-461777fb6f67 // indirect
+	golang.org/x/sys v0.0.0-20190610081024-1e42afee0f76 // indirect
+	golang.org/x/tools v0.0.0-20190610190622-bca362e842d4 // indirect
 	google.golang.org/api v0.6.0
 	google.golang.org/appengine v1.6.1 // indirect
 )

go.sum

@@ -221,6 +221,8 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0O
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190606173856-1492cefac77f h1:IWHgpgFqnL5AhBUBZSgBdjl2vkQUEzcY+JNKWfcgAU0=
 golang.org/x/net v0.0.0-20190606173856-1492cefac77f/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190607181551-461777fb6f67 h1:rJJxsykSlULwd2P2+pg/rtnwN2FrWp4IuCxOSyS0V00=
+golang.org/x/net v0.0.0-20190607181551-461777fb6f67/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -259,6 +261,8 @@ golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444 h1:/d2cWp6PSamH4jDPFLyO150psQdqvtoNX8Zjg3AQ31g=
 golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190610081024-1e42afee0f76 h1:QSmW7Q3mFdAGjtAd0byXmFJ55inUydyZ4WQmiuItAIQ=
+golang.org/x/sys v0.0.0-20190610081024-1e42afee0f76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -281,6 +285,8 @@ golang.org/x/tools v0.0.0-20190529010454-aa71c3f32488/go.mod h1:RgjU9mgBXZiqYHBn
 golang.org/x/tools v0.0.0-20190530171427-2b03ca6e44eb/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190603231351-8aaa1484dc10/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190610190622-bca362e842d4 h1:mlrypYegmpBUm3gXAR2XkF+d8w2b6xGzTmuTERlpUyU=
+golang.org/x/tools v0.0.0-20190610190622-bca362e842d4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
 google.golang.org/api v0.3.2 h1:iTp+3yyl/KOtxa/d1/JUE0GGSoR6FuW5udver22iwpw=
 google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=

youtubedl.go

@@ -380,9 +380,19 @@ func GetPlaylistVideos(playListID string, apiKey string) (members []string, err
 	return members, nil
 }
 
-// LoadYouTubeAnnotationsVideo downloads the subtitles of the specified YouTube video and creates annotations in the specified mongo db collection
+// LoadYouTubeVideos downloads the subtitles of the specified YouTube videos and creates annotations in the specified mongo db collection
 // user supplied YouTube subtitles are preferred over auto-generated ones
-func LoadYouTubeAnnotationsVideo(videoID string, language string, collection *mongo.Collection) error {
+func LoadYouTubeVideos(videoIDs []string, language string, collection *mongo.Collection) error {
+	for _, v := range videoIDs {
+		err := loadYouTubeAnnotationsVideo(v, language, collection)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func loadYouTubeAnnotationsVideo(videoID string, language string, collection *mongo.Collection) error {
 	var subs annotation.Caption
 	err := errors.New("")
 
@@ -425,7 +435,7 @@ func LoadYouTubeAnnotationsPlaylist(playlistID string, language string, collecti
 	log.WithFields(log.Fields{"playlistID": playlistID, "language": language, "quantity": len(members)}).Trace("Videos found in playlist")
 
 	for _, videoID := range members {
-		err := LoadYouTubeAnnotationsVideo(videoID, language, collection)
+		err := loadYouTubeAnnotationsVideo(videoID, language, collection)
 		if err != nil {
 			log.Warning(err)
 			continue
@@ -507,7 +517,7 @@ func fromTimedText(subtitle *TimedText, minWindowLength int, maxWindowLength int
 	for paragraphNum, paragraph := range subtitle.Body.Paragraphs {
 
 		// Handle manual captions which have no segments (no word granularity timing)
-		if len(paragraph.Segments) == 0 && paragraph.Value != "" {
+		if len(paragraph.Segments) == 0 && strings.Join(strings.Fields(paragraph.Value), " ") != "" {
 
 			// Create annotation for the paragraph
 			a := &annotation.Annotation{
@@ -541,9 +551,26 @@ func fromTimedText(subtitle *TimedText, minWindowLength int, maxWindowLength int
 
 				// Compute the start time of the current window
 				startSegment := paragraph.Segments[startSegmentIdx]
-				endSegment := paragraph.Segments[startSegmentIdx+wLength-1]
 				startTime := paragraph.Time + startSegment.Time
-				endTime := paragraph.Time + endSegment.Time + endSegment.Duration
+
+				var endTime int
+				// if current segment not last one
+				if startSegmentIdx+wLength < len(paragraph.Segments) {
+					nextSegment := paragraph.Segments[startSegmentIdx+wLength]
+					endTime = paragraph.Time + nextSegment.Time
+				} else {
+					if paragraphNum < len(subtitle.Body.Paragraphs)-1 {
+						// handle when current segment/paragraph subtitle stays displayed even when next one appears
+						if paragraph.Time+paragraph.Duration > subtitle.Body.Paragraphs[paragraphNum+1].Time {
+							endTime = subtitle.Body.Paragraphs[paragraphNum+1].Time
+						} else {
+							endTime = paragraph.Time + paragraph.Duration
+						}
+
+					} else {
+						endTime = paragraph.Time + paragraph.Duration
+					}
+				}
 
 				// Create annotation of the current window
 				a := &annotation.Annotation{

youtubedl_test.go

@@ -182,7 +182,7 @@ func TestTimedTextToAnnotationAutomaticCaption(t *testing.T) {
 			Count: 0,
 			Text:  "you're",
 			Start: 170,
-			End:   372,
+			End:   1429,
 		},
 	}
 	assert.Equal(t, aExpect, *a[0])
@@ -193,7 +193,7 @@ func TestTimedTextToAnnotationAutomaticCaption(t *testing.T) {
 			Count: 0,
 			Text:  "we",
 			Start: 2970,
-			End:   3222,
+			End:   3510,
 		},
 	}
 	assert.Equal(t, aExpect, *a[15])
@@ -202,12 +202,24 @@ func TestTimedTextToAnnotationAutomaticCaption(t *testing.T) {
 		Src: "youtube://VIlLpnJJl_4",
 		Subtitle: annotation.Subtitle{
 			Count: 2,
-			Text:  "12 hours",
+			Text:  "12 hours a",
 			Start: 6120,
-			End:   6672,
+			End:   6930,
 		},
 	}
 	assert.Equal(t, aExpect, *a[32])
+
+	// test endTime of last segment of paragraph
+	aExpect = annotation.Annotation{
+		Src: "youtube://VIlLpnJJl_4",
+		Subtitle: annotation.Subtitle{
+			Count: 2,
+			Text:  "just to",
+			Start: 7140,
+			End:   7970,
+		},
+	}
+	assert.Equal(t, aExpect, *a[43])
 }
 
 func TestNumberOfProducedAnnotations(t *testing.T) {