To receive notifications about scheduled maintenance, please subscribe to the mailing-list gitlab-operations@sympa.ethz.ch. You can subscribe to the mailing-list at https://sympa.ethz.ch

Commit 08e094d2 authored by Christof Gerber's avatar Christof Gerber
Browse files

Make loading multiple videos less verbose. endTime from next segment/paragraph

parent df7880a2
Pipeline #36090 passed with stages
in 6 minutes and 30 seconds
......@@ -11,8 +11,9 @@ require (
gitlab.ethz.ch/chgerber/annotation v0.0.0-20190607145645-0049de6d439e
gitlab.ethz.ch/chgerber/monitor v0.0.0-20190527191251-2bb9dd731340
go.mongodb.org/mongo-driver v1.0.3
golang.org/x/net v0.0.0-20190606173856-1492cefac77f // indirect
golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444 // indirect
golang.org/x/net v0.0.0-20190607181551-461777fb6f67 // indirect
golang.org/x/sys v0.0.0-20190610081024-1e42afee0f76 // indirect
golang.org/x/tools v0.0.0-20190610190622-bca362e842d4 // indirect
google.golang.org/api v0.6.0
google.golang.org/appengine v1.6.1 // indirect
)
......@@ -221,6 +221,8 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0O
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190606173856-1492cefac77f h1:IWHgpgFqnL5AhBUBZSgBdjl2vkQUEzcY+JNKWfcgAU0=
golang.org/x/net v0.0.0-20190606173856-1492cefac77f/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190607181551-461777fb6f67 h1:rJJxsykSlULwd2P2+pg/rtnwN2FrWp4IuCxOSyS0V00=
golang.org/x/net v0.0.0-20190607181551-461777fb6f67/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
......@@ -259,6 +261,8 @@ golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444 h1:/d2cWp6PSamH4jDPFLyO150psQdqvtoNX8Zjg3AQ31g=
golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190610081024-1e42afee0f76 h1:QSmW7Q3mFdAGjtAd0byXmFJ55inUydyZ4WQmiuItAIQ=
golang.org/x/sys v0.0.0-20190610081024-1e42afee0f76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
......@@ -281,6 +285,8 @@ golang.org/x/tools v0.0.0-20190529010454-aa71c3f32488/go.mod h1:RgjU9mgBXZiqYHBn
golang.org/x/tools v0.0.0-20190530171427-2b03ca6e44eb/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190603231351-8aaa1484dc10/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190610190622-bca362e842d4 h1:mlrypYegmpBUm3gXAR2XkF+d8w2b6xGzTmuTERlpUyU=
golang.org/x/tools v0.0.0-20190610190622-bca362e842d4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
google.golang.org/api v0.3.2 h1:iTp+3yyl/KOtxa/d1/JUE0GGSoR6FuW5udver22iwpw=
google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
......
......@@ -380,9 +380,19 @@ func GetPlaylistVideos(playListID string, apiKey string) (members []string, err
return members, nil
}
// LoadYouTubeAnnotationsVideo downloads the subtitles of the specified YouTube video and creates annotations in the specified mongo db collection
// LoadYouTubeVideos downloads the subtitles of the specified YouTube videos and creates annotations in the specified mongo db collection
// user supplied YouTube subtitles are preferred over auto-generated ones
func LoadYouTubeAnnotationsVideo(videoID string, language string, collection *mongo.Collection) error {
func LoadYouTubeVideos(videoIDs []string, language string, collection *mongo.Collection) error {
for _, v := range videoIDs {
err := loadYouTubeAnnotationsVideo(v, language, collection)
if err != nil {
return err
}
}
return nil
}
func loadYouTubeAnnotationsVideo(videoID string, language string, collection *mongo.Collection) error {
var subs annotation.Caption
err := errors.New("")
......@@ -425,7 +435,7 @@ func LoadYouTubeAnnotationsPlaylist(playlistID string, language string, collecti
log.WithFields(log.Fields{"playlistID": playlistID, "language": language, "quantity": len(members)}).Trace("Videos found in playlist")
for _, videoID := range members {
err := LoadYouTubeAnnotationsVideo(videoID, language, collection)
err := loadYouTubeAnnotationsVideo(videoID, language, collection)
if err != nil {
log.Warning(err)
continue
......@@ -507,7 +517,7 @@ func fromTimedText(subtitle *TimedText, minWindowLength int, maxWindowLength int
for paragraphNum, paragraph := range subtitle.Body.Paragraphs {
// Handle manual captions which have no segments (no word granularity timing)
if len(paragraph.Segments) == 0 && paragraph.Value != "" {
if len(paragraph.Segments) == 0 && strings.Join(strings.Fields(paragraph.Value), " ") != "" {
// Create annotation for the paragraph
a := &annotation.Annotation{
......@@ -541,9 +551,26 @@ func fromTimedText(subtitle *TimedText, minWindowLength int, maxWindowLength int
// Compute the start time of the current window
startSegment := paragraph.Segments[startSegmentIdx]
endSegment := paragraph.Segments[startSegmentIdx+wLength-1]
startTime := paragraph.Time + startSegment.Time
endTime := paragraph.Time + endSegment.Time + endSegment.Duration
var endTime int
// if current segment not last one
if startSegmentIdx+wLength < len(paragraph.Segments) {
nextSegment := paragraph.Segments[startSegmentIdx+wLength]
endTime = paragraph.Time + nextSegment.Time
} else {
if paragraphNum < len(subtitle.Body.Paragraphs)-1 {
// handle when current segment/paragraph subtitle stays displayed even when next one appears
if paragraph.Time+paragraph.Duration > subtitle.Body.Paragraphs[paragraphNum+1].Time {
endTime = subtitle.Body.Paragraphs[paragraphNum+1].Time
} else {
endTime = paragraph.Time + paragraph.Duration
}
} else {
endTime = paragraph.Time + paragraph.Duration
}
}
// Create annotation of the current window
a := &annotation.Annotation{
......
......@@ -182,7 +182,7 @@ func TestTimedTextToAnnotationAutomaticCaption(t *testing.T) {
Count: 0,
Text: "you're",
Start: 170,
End: 372,
End: 1429,
},
}
assert.Equal(t, aExpect, *a[0])
......@@ -193,7 +193,7 @@ func TestTimedTextToAnnotationAutomaticCaption(t *testing.T) {
Count: 0,
Text: "we",
Start: 2970,
End: 3222,
End: 3510,
},
}
assert.Equal(t, aExpect, *a[15])
......@@ -202,12 +202,24 @@ func TestTimedTextToAnnotationAutomaticCaption(t *testing.T) {
Src: "youtube://VIlLpnJJl_4",
Subtitle: annotation.Subtitle{
Count: 2,
Text: "12 hours",
Text: "12 hours a",
Start: 6120,
End: 6672,
End: 6930,
},
}
assert.Equal(t, aExpect, *a[32])
// test endTime of last segment of paragraph
aExpect = annotation.Annotation{
Src: "youtube://VIlLpnJJl_4",
Subtitle: annotation.Subtitle{
Count: 2,
Text: "just to",
Start: 7140,
End: 7970,
},
}
assert.Equal(t, aExpect, *a[43])
}
func TestNumberOfProducedAnnotations(t *testing.T) {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment