testconfig_validator.py 35.8 KB
Newer Older
Reto Da Forno's avatar
Reto Da Forno committed
1
#!/usr/bin/env python3
2

3
import sys, os, getopt, errno, subprocess, time, calendar, MySQLdb, tempfile, base64, syslog, re, configparser, traceback, lxml.etree, logging
4
import lib.flocklab as flocklab
5
6


7
debug = False
8
9
10
11


##############################################################################
#
12
# check_obsids
Reto Da Forno's avatar
Reto Da Forno committed
13
14
#    Checks if every observer ID returned by the xpath evaluation is only used
#    once and if every observer ID is in the list provided in obsidlist.
15
16
#
##############################################################################
17
def check_obsids(tree, xpathExpr, namespace, obsidlist=None):
Reto Da Forno's avatar
Reto Da Forno committed
18
19
20
21
22
23
24
25
26
27
28
29
30
    duplicates = False
    allInList  = True
    
    # Get the observer IDs from the xpath expression:
    rs = tree.xpath(xpathExpr, namespaces=namespace)
    
    # Build a list with all used observer IDs in it:
    foundObsids = []
    tmp = []
    for ids in rs:
        tmp.append(ids.text.split())
    list(map(foundObsids.extend, tmp))
    
Reto Da Forno's avatar
Reto Da Forno committed
31
32
33
    if "ALL" in foundObsids:
        return (obsidlist, duplicates, allInList)
    
Reto Da Forno's avatar
Reto Da Forno committed
34
35
36
37
38
39
40
41
42
43
44
    # Check for duplicates:
    if ( (len(foundObsids) != len(set(foundObsids))) ):
        duplicates = True
    
    # Check if all obs ids are in the list:
    for obsid in foundObsids:
        if obsid not in obsidlist:
            allInList = False
    
    # Return the values to the caller:
    if (duplicates or not allInList):
Reto Da Forno's avatar
Reto Da Forno committed
45
        return (None, duplicates, allInList)
Reto Da Forno's avatar
Reto Da Forno committed
46
    else:
Reto Da Forno's avatar
Reto Da Forno committed
47
        return (sorted(foundObsids), duplicates, allInList)
48
### END check_obsids()
49
50
51
52
53
54
55


##############################################################################
#
# Usage
#
##############################################################################
Reto Da Forno's avatar
Reto Da Forno committed
56
def usage():
57
    print("Usage: %s [--xml=<path>] [--testid=<int>] [--userid=<int>] [--schema=<path>] [--quiet] [--help]" % sys.argv[0])
Reto Da Forno's avatar
Reto Da Forno committed
58
59
60
61
62
    print("Validate an XML testconfiguration. Returns 0 on success, errno on errors.")
    print("Options:")
    print("  --xml\t\t\t\tOptional. Path to the XML file which is to check. Either --xml or --testid are) mandatory. If both are given, --testid will be favoured.")
    print("  --testid\t\t\tOptional. Test ID to validate. If this parameter is set, the XML will be taken from the DB. Either --xml or --testid are mandatory. If both are given, --testid will be favoured.")
    print("  --userid\t\t\tOptional. User ID to which the XML belongs. Mandatory if --xml is specified.")
Reto Da Forno's avatar
Reto Da Forno committed
63
    print("  --schema\t\t\tOptional. Path to the XML schema to check XML against. If not given, the standard path will be used: %s" %(str(flocklab.config.get('xml', 'schemapath'))))
Reto Da Forno's avatar
Reto Da Forno committed
64
65
    print("  --quiet\t\t\tOptional. Do not print on standard out.")
    print("  --help\t\t\tOptional. Print this help.")
66
67
68
69
70
71
72
73
74
### END usage()


##############################################################################
#
# Main
#
##############################################################################
def main(argv):
75
76
77
78
79
    quiet      = False
    userid     = None
    xmlpath    = None
    schemapath = None
    testid     = None
Reto Da Forno's avatar
Reto Da Forno committed
80
    userrole   = ""
Reto Da Forno's avatar
Reto Da Forno committed
81
82
    
    # Open the log and create logger:
83
    logger = flocklab.get_logger(debug=debug)
Reto Da Forno's avatar
Reto Da Forno committed
84
85
    
    # Get the config file:
86
    flocklab.load_config()
Reto Da Forno's avatar
Reto Da Forno committed
87
88
89
90
91
92
    
    # Get command line parameters.
    try:
        opts, args = getopt.getopt(argv, "hqu:s:x:t:", ["help", "quiet", "userid=", "schema=", "xml=", "testid="])
    except getopt.GetoptError as err:
        logger.warn(str(err))
Reto Da Forno's avatar
Reto Da Forno committed
93
        usage()
Reto Da Forno's avatar
Reto Da Forno committed
94
95
96
97
98
99
        sys.exit(errno.EINVAL)
    for opt, arg in opts:
        if opt in ("-u", "--userid"):
            try:
                userid = int(arg)
                if userid <= 0:
100
                    raise
Reto Da Forno's avatar
Reto Da Forno committed
101
102
103
104
105
106
107
            except:
                logger.warn("Wrong API usage: userid has to be a positive number")
                sys.exit(errno.EINVAL)
        elif opt in ("-t", "--testid"):
            try:
                testid = int(arg)
                if testid <= 0:
108
                    raise
Reto Da Forno's avatar
Reto Da Forno committed
109
110
111
112
113
114
115
116
117
118
119
            except:
                logger.warn("Wrong API usage: testid has to be a positive number")
                sys.exit(errno.EINVAL)
        elif opt in ("-s", "--schema"):
            schemapath = arg
            if (not os.path.exists(schemapath) or not os.path.isfile(schemapath)):
                logger.warn("Wrong API usage: schema file '%s' does not exist" % schemapath)
                sys.exit(errno.EINVAL)
        elif opt in ("-x", "--xml"):
            xmlpath = arg
            if (not os.path.exists(xmlpath) or not os.path.isfile(xmlpath)):
120
                logger.warn("Wrong API usage: XML file '%s' does not exist" % xmlpath)
Reto Da Forno's avatar
Reto Da Forno committed
121
122
                sys.exit(errno.EINVAL)
        elif opt in ("-h", "--help"):
Reto Da Forno's avatar
Reto Da Forno committed
123
            usage()
Reto Da Forno's avatar
Reto Da Forno committed
124
125
126
127
128
129
            sys.exit(SUCCESS)
        elif opt in ("-q", "--quiet"):
            quiet = True
        else:
            if not quiet:
                print("Wrong API usage")
Reto Da Forno's avatar
Reto Da Forno committed
130
                usage()
Reto Da Forno's avatar
Reto Da Forno committed
131
132
133
134
135
136
137
            logger.warn("Wrong API usage")
            sys.exit(errno.EINVAL)
    
    # Check mandatory arguments:
    if ( ((not testid) and (not xmlpath)) or ((xmlpath) and (not userid)) ):
        if not quiet:
            print("Wrong API usage")
Reto Da Forno's avatar
Reto Da Forno committed
138
            usage()
Reto Da Forno's avatar
Reto Da Forno committed
139
140
141
142
143
        logger.warn("Wrong API usage")
        sys.exit(errno.EINVAL)
    
    # Set the schemapath:
    if not schemapath:
Reto Da Forno's avatar
Reto Da Forno committed
144
        schemapath = flocklab.config.get('xml', 'schemapath')
Reto Da Forno's avatar
Reto Da Forno committed
145
146
147
148
149
150
151
152
153
154
155
156
157
    
    # check if xmllint is installed
    try:
        subprocess.check_call(['which', 'xmllint'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        # if check_call doesn't raise an exception, then return code was zero (success)
    except:
        if not quiet:
            print("xmllint not found!")
        logger.warn("xmllint not found!")
        sys.exit(errno.EINVAL)
    
    # Connect to the DB:
    try:
158
        (cn, cur) = flocklab.connect_to_db()
Reto Da Forno's avatar
Reto Da Forno committed
159
    except:
160
161
        flocklab.error_logandexit("Could not connect to database", errno.EAGAIN)
    
Reto Da Forno's avatar
Reto Da Forno committed
162
    # Check if the user is admin:
163
164
165
    if userid is None:
        userid = flocklab.get_test_owner(cur, testid)[0]
    userrole = flocklab.get_user_role(cur, userid)
Reto Da Forno's avatar
Reto Da Forno committed
166
    if userrole is None:
167
        logger.warn("Could not determine role for user ID %d." % userid)
Reto Da Forno's avatar
Reto Da Forno committed
168
        sys.exit(errno.EAGAIN)
169
170
171
    
    # Valid stati for observers based on used permissions
    stati = "'online'"
Reto Da Forno's avatar
Reto Da Forno committed
172
    if "admin" in userrole:
173
        stati += ", 'develop', 'internal'"
Reto Da Forno's avatar
Reto Da Forno committed
174
    elif "internal" in userrole:
175
176
        stati += ", 'internal'"
    
Reto Da Forno's avatar
Reto Da Forno committed
177
178
179
    # Initialize error counter and set timezone to UTC:
    errcnt = 0;
    
180
    logger.debug("Checking XML config...")
Reto Da Forno's avatar
Reto Da Forno committed
181
182
183
184
185
186
187
    
    #===========================================================================
    # If a testid was given, get the xml from the database
    #===========================================================================
    if testid:
        # Get the XML from the database, put it into a temp file and set the xmlpath accordingly:
        (fd, xmlpath) = tempfile.mkstemp()
188
189
        cur.execute("SELECT `testconfig_xml`, `owner_fk` FROM `tbl_serv_tests` WHERE (`serv_tests_key` = %s)" %testid)
        ret = cur.fetchone()
Reto Da Forno's avatar
Reto Da Forno committed
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
        if not ret:
            if not quiet:
                print(("No test found in database with testid %d. Exiting..." %testid))
            errcnt = errcnt + 1
        else:
            xmlfile = os.fdopen(fd, 'w+')
            xmlfile.write(ret[0])
            xmlfile.close()
            userid = int(ret[1])
    
    #===========================================================================
    # Validate the XML against the XML schema
    #===========================================================================
    if errcnt == 0:
        try:
            p = subprocess.Popen(['xmllint', '--noout', xmlpath, '--schema', schemapath], stdout=subprocess.PIPE, stderr= subprocess.PIPE, universal_newlines=True)
            stdout, stderr = p.communicate()
            for err in stderr.split('\n'):
                tmp = err.split(':')
                if len(tmp) >= 7:
                    if not quiet:
211
                        print(("Line " + tmp[1] + ":" + tmp[2] + ":" + ":".join(tmp[6:])))
Reto Da Forno's avatar
Reto Da Forno committed
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
                    errcnt = errcnt + 1
                elif not ((err.find('fails to validate') != -1) or (err.find('validates') != -1) or (err == '\n') or (err == '')):
                    if not quiet:
                        print(err)
                    errcnt = errcnt + 1
        except:
            exc_type, exc_obj, exc_tb = sys.exc_info()
            fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
            print(("%s %s, %s %s" % (exc_type, sys.exc_info()[1], fname, exc_tb.tb_lineno)))
            errcnt = errcnt + 1
    
    #===========================================================================
    # If XML is valid, do additional checks on <generalConf> and <targetConf> elements
    #===========================================================================
    if errcnt == 0:
        # generalConf additional validation -------------------------------------------
        #    * If specified, start time has to be in the future
        #    * If specified, end time has to be after start time
        f = open(xmlpath, 'r')
231
232
        parser = lxml.etree.XMLParser(remove_comments=True)
        tree = lxml.etree.parse(f, parser)
Reto Da Forno's avatar
Reto Da Forno committed
233
        f.close()
Reto Da Forno's avatar
Reto Da Forno committed
234
        ns = {'d': flocklab.config.get('xml', 'namespace')}
Reto Da Forno's avatar
Reto Da Forno committed
235
236
237
238
239
240
241
242
243
244
245
246
247
        # additional check for the namespace
        m = re.match('\{.*\}', tree.getroot().tag)
        if not m:
            print("Failed to extract namespace from XML file.")
            errcnt = errcnt + 1
        m = m.group(0)[1:-1]  # remove braces
        if m != ns['d']:
            print("Namespace in XML file does not match: found '%s', expected '%s'." % (m, ns['d']))
            errcnt = errcnt + 1
        # check xs:list items (obsIds, targetIds) for whitespace as separator
        for l in tree.xpath('//d:*/d:obsIds', namespaces=ns) + tree.xpath('//d:*/d:targetIds', namespaces=ns):
            if l.text.find('\t')>=0:
                if not quiet:
248
                    print("Element obsIds/targetIds: Id lists must not have tabs as separators.")
Reto Da Forno's avatar
Reto Da Forno committed
249
250
251
252
253
254
255
256
257
                errcnt = errcnt + 1
    
    if errcnt == 0:
        sched_abs  = tree.xpath('//d:generalConf/d:scheduleAbsolute', namespaces=ns)
        sched_asap = tree.xpath('//d:generalConf/d:scheduleAsap', namespaces=ns)
        if sched_abs:
            # The start date and time have to be in the future:
            rs = tree.xpath('//d:generalConf/d:scheduleAbsolute/d:start', namespaces=ns)
            now = time.time()
258
            testStart = flocklab.get_xml_timestamp(rs[0].text)
Reto Da Forno's avatar
Reto Da Forno committed
259
260
            if (testStart <= now):
                if not quiet:
261
                    print("Element generalConf: Start time has to be in the future.")
Reto Da Forno's avatar
Reto Da Forno committed
262
263
264
                errcnt = errcnt + 1
            # The end date and time have to be in the future and after the start:
            rs = tree.xpath('//d:generalConf/d:scheduleAbsolute/d:end', namespaces=ns)
265
            testEnd = flocklab.get_xml_timestamp(rs[0].text)
Reto Da Forno's avatar
Reto Da Forno committed
266
267
            if (testEnd <= testStart):
                if not quiet:
268
                    print("Element generalConf: End time has to be after start time.")
Reto Da Forno's avatar
Reto Da Forno committed
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
                errcnt = errcnt + 1
            # Calculate the test duration which is needed later on:
            testDuration = testEnd - testStart
        elif sched_asap:
            testDuration = int(tree.xpath('//d:generalConf/d:scheduleAsap/d:durationSecs', namespaces=ns)[0].text)
        
        # targetConf additional validation -------------------------------------------- 
        #    * DB image ids need to be in the database and binary field must not be empty
        #    * Embedded image ids need to be in elements in the XML and need to be valid and correct type
        #    * Observer ids need to have the correct target adaptor installed and must be unique
        #    * If specified, number of target ids need to be the same as observer ids
        #    * There must be a target image provided for every mandatory core (usually core 0, core 0-3 for DPP)
        
        # Loop through all targetConf elements:
        obsidlist = []
        obsiddict = {}
        targetconfs = tree.xpath('//d:targetConf', namespaces=ns)
        for targetconf in targetconfs:
            targetids = None
            dbimageid = None
            embimageid = None
290
            platform = None
Reto Da Forno's avatar
Reto Da Forno committed
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
            # Get elements:
            obsids = targetconf.xpath('d:obsIds', namespaces=ns)[0].text.split()
            ret = targetconf.xpath('d:targetIds', namespaces=ns)
            if ret:
                targetids = ret[0].text.split()
                targetids_line = ret[0].sourceline 
            ret = targetconf.xpath('d:dbImageId', namespaces=ns)
            if ret:
                dbimageid = [o.text for o in ret]
                dbimageid_line = [o.sourceline for o in ret]
            ret = targetconf.xpath('d:embeddedImageId', namespaces=ns)
            if ret:
                embimageid = [o.text for o in ret]
                embimageid_line = [o.sourceline for o in ret]
            
            # If target ids are present, there need to be as many as observer ids:
            if (targetids and (len(targetids) != len(obsids))):
                if not quiet:
309
                    print(("Line %d: element targetIds: If element targetIds is used, it needs the same amount of IDs as in the corresponding element obsIds." %(targetids_line)))
Reto Da Forno's avatar
Reto Da Forno committed
310
311
312
313
314
315
316
317
318
319
320
321
                errcnt = errcnt + 1
            
            # If DB image IDs are present, check if they are in the database and belong to the user (if he is not an admin) and get values for later use:
            if dbimageid:
                for dbimg, line in zip(dbimageid, dbimageid_line):
                    sql = """    SELECT b.name, c.name, a.core
                                FROM `tbl_serv_targetimages` AS a 
                                LEFT JOIN `tbl_serv_operatingsystems` AS b 
                                    ON a.operatingsystems_fk = b.serv_operatingsystems_key 
                                LEFT JOIN `tbl_serv_platforms` AS c 
                                    ON a.platforms_fk = c.serv_platforms_key
                                WHERE (a.`serv_targetimages_key` = %s AND a.`binary` IS NOT NULL)""" %(dbimg)
Reto Da Forno's avatar
Reto Da Forno committed
322
                    if "admin" not in userrole:
Reto Da Forno's avatar
Reto Da Forno committed
323
                        sql += " AND (a.`owner_fk` = %s)"%(userid)
324
325
                    cur.execute(sql)
                    ret = cur.fetchone()
Reto Da Forno's avatar
Reto Da Forno committed
326
327
                    if not ret:
                        if not quiet:
328
                            print(("Line %d: element dbImageId: The image with ID %s does not exist in the database or does not belong to you." %(line, str(dbimg))))
Reto Da Forno's avatar
Reto Da Forno committed
329
330
331
332
333
334
335
336
337
                        errcnt = errcnt + 1
                    else:
                        # Put data into dictionary for later use:
                        core = int(ret[2])
                        for obsid in obsids:
                            if obsid not in obsiddict:
                                obsiddict[obsid] = {}
                            if core in obsiddict[obsid]:
                                if not quiet:
338
                                    print(("Line %d: element dbImageId: There is already an image for core %d (image with ID %s)." %(line, core, str(dbimg))))
Reto Da Forno's avatar
Reto Da Forno committed
339
340
341
342
343
344
345
346
347
348
                                errcnt = errcnt + 1
                            else:
                                obsiddict[obsid][core]=ret[:2]
            
            # If embedded image IDs are present, check if they have a corresponding <imageConf> which is valid:
            if embimageid:
                for embimg, line in zip(embimageid, embimageid_line):
                    imageconf = tree.xpath('//d:imageConf/d:embeddedImageId[text()="%s"]/..' %(embimg), namespaces=ns)
                    if not imageconf:
                        if not quiet:
349
                            print(("Line %d: element embeddedImageId: There is no corresponding element imageConf with embeddedImageId %s defined." %(line, embimg)))
Reto Da Forno's avatar
Reto Da Forno committed
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
                        errcnt = errcnt + 1
                    else:
                        # Get os and platform and put it into dictionary for later use:
                        if imageconf[0].xpath('d:os', namespaces=ns):
                            opersys = imageconf[0].xpath('d:os', namespaces=ns)[0].text
                        else:
                            opersys = 'other'
                        platform = imageconf[0].xpath('d:platform', namespaces=ns)[0].text
                        try:
                            core = int(imageconf[0].xpath('d:core', namespaces=ns)[0].text)
                        except:
                            # not a mandatory field, use the default value
                            core = 0
                        logger.debug("Target image for platform %s (core %d) found." % (platform, core))
                        for obsid in obsids:
                            if obsid not in obsiddict:
                                obsiddict[obsid] = {}
                            if core in obsiddict[obsid]:
                                if not quiet:
369
                                    print(("Line %d: element dbImageId: There is already an image for core %d (image with ID %s)." %(line, core, str(embimg))))
Reto Da Forno's avatar
Reto Da Forno committed
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
                                errcnt = errcnt + 1
                            obsiddict[obsid][core]=(opersys, platform)
                        # Get the image and save it to a temporary file:
                        image = imageconf[0].xpath('d:data', namespaces=ns)[0].text
                        # For target platform DPP2LoRa, the <data> tag may be empty
                        if len(image.strip()) == 0 and (platform.lower() == "dpp2lora" or platform.lower() == "dpp2lorahg"):
                            continue   # skip image validation
                        image_line = imageconf[0].xpath('d:data', namespaces=ns)[0].sourceline
                        (fd, imagefilename) = tempfile.mkstemp()
                        imagefile = os.fdopen(fd, 'w+b')
                        if not imagefile:
                            print("Failed to create file %s." % (imagefilename))
                        imagefile.write(base64.b64decode(image, None))
                        imagefile.close()
                        # Validate image:
Reto Da Forno's avatar
Reto Da Forno committed
385
                        p = subprocess.Popen([flocklab.config.get('targetimage', 'imagevalidator'), '--quiet', '--image', imagefilename, '--platform', platform], stderr=subprocess.PIPE, universal_newlines=True)
Reto Da Forno's avatar
Reto Da Forno committed
386
                        stdout, stderr = p.communicate()
Reto Da Forno's avatar
Reto Da Forno committed
387
                        if p.returncode != flocklab.SUCCESS:
Reto Da Forno's avatar
Reto Da Forno committed
388
                            if not quiet:
389
                                print(("Line %d: element data: Validation of image data failed. %s" %(image_line, stderr)))
Reto Da Forno's avatar
Reto Da Forno committed
390
391
392
                            errcnt = errcnt + 1
                        # Remove temporary file:
                        os.remove(imagefilename)
393
394
395
396
397
398
399
400
401
402
403
            
            # Put obsids into obsidlist:
            for obsid in obsids:
                if obsid == "ALL":
                    # expand
                    rs = flocklab.get_obsids(cur, platform, stati)
                    if rs:
                        obsidlist.extend(rs)
                else:
                    obsidlist.append(obsid)
                
404
        # if there is just one target config and a list of observers is not provided, then fetch a list of all observers from the database
Reto Da Forno's avatar
Reto Da Forno committed
405
406
407
        if not obsidlist or (len(obsidlist) == 0):
            print("No observer ID found.")
            errcnt = errcnt + 1
408
        # Remove duplicates and check if every observer has the correct target adapter installed:
Reto Da Forno's avatar
Reto Da Forno committed
409
        obsidlist = list(set(obsidlist))
410
        (obsids, duplicates, allInList) = check_obsids(tree, '//d:targetConf/d:obsIds', ns, obsidlist)
Reto Da Forno's avatar
Reto Da Forno committed
411
412
        if duplicates:
            if not quiet:
413
                print("Element targetConf: Some observer IDs have been used more than once.")
Reto Da Forno's avatar
Reto Da Forno committed
414
415
416
417
418
419
420
421
422
423
            errcnt = errcnt + 1
        else:
            usedObsidsList = sorted(obsids)
            # Now that we have the list, check the observer types:
            sql_adap = """SELECT `b`.`tg_adapt_types_fk`, `c`.`tg_adapt_types_fk`, `d`.`tg_adapt_types_fk`, `e`.`tg_adapt_types_fk`
                            FROM `tbl_serv_observer` AS `a` 
                            LEFT JOIN `tbl_serv_tg_adapt_list` AS `b` ON `a`.`slot_1_tg_adapt_list_fk` = `b`.`serv_tg_adapt_list_key`
                            LEFT JOIN `tbl_serv_tg_adapt_list` AS `c` ON `a`.`slot_2_tg_adapt_list_fk` = `c`.`serv_tg_adapt_list_key`
                            LEFT JOIN `tbl_serv_tg_adapt_list` AS `d` ON `a`.`slot_3_tg_adapt_list_fk` = `d`.`serv_tg_adapt_list_key`
                            LEFT JOIN `tbl_serv_tg_adapt_list` AS `e` ON `a`.`slot_4_tg_adapt_list_fk` = `e`.`serv_tg_adapt_list_key`
Reto Da Forno's avatar
Reto Da Forno committed
424
425
                            WHERE (`a`.`observer_id` = %s)
                            AND (`a`.`status` IN (%s))
Reto Da Forno's avatar
Reto Da Forno committed
426
427
428
429
                        """
            sql_platf = """SELECT COUNT(*)
                            FROM `tbl_serv_tg_adapt_types` AS `a` 
                            LEFT JOIN `tbl_serv_platforms` AS `b` ON `a`.`platforms_fk` = `b`.`serv_platforms_key`
Reto Da Forno's avatar
Reto Da Forno committed
430
431
                            WHERE (`a`.`serv_tg_adapt_types_key` = %s)
                            AND (LOWER(`b`.`name`) = LOWER('%s'))
Reto Da Forno's avatar
Reto Da Forno committed
432
433
434
435
436
437
438
439
440
441
442
443
444
                        """
            sql_cores = """SELECT core, optional
                            FROM `tbl_serv_platforms` AS `b` 
                            LEFT JOIN `tbl_serv_architectures` AS `a` ON `a`.`platforms_fk` = `b`.`serv_platforms_key`
                            WHERE (LOWER(`b`.`name`) = LOWER('%s'))
                        """
            for obsid in usedObsidsList:
                if obsid in obsiddict:
                    platf = next(iter(obsiddict[obsid].values()))[1].lower()
                    opersys = next(iter(obsiddict[obsid].values()))[0].lower()
                    for p in obsiddict[obsid].values():
                        if platf!=p[1].lower():
                            if not quiet:
445
                                print(("Element targetConf: Observer ID %s has images of several platform types assigned." %(obsid)))
Reto Da Forno's avatar
Reto Da Forno committed
446
447
448
449
                            errcnt = errcnt + 1
                            break
                        #if opersys!=p[0].lower():
                        #    if not quiet:
450
                        #        print(("Element targetConf: Observer ID %s has images of several operating system types assigned." %(obsid)))
Reto Da Forno's avatar
Reto Da Forno committed
451
452
453
454
455
                        #    errcnt = errcnt + 1
                        #    break
                else:
                    platf = None
                # Get tg_adapt_types_fk of installed target adaptors on observer:
456
457
                cur.execute(sql_adap %(obsid, stati))
                adaptTypesFk = cur.fetchone()
458
                # If no results are returned, it most probably means that the observer is not active at the moment.
Reto Da Forno's avatar
Reto Da Forno committed
459
460
                if not adaptTypesFk:
                    if not quiet:
461
462
463
464
                        print(("Element targetConf: Observer ID %s cannot be used at the moment." %(obsid)))
                    # If the test ID has been provided, the test has already been scheduled and should run despite a node that is not available.
                    if not testid:
                        errcnt = errcnt + 1
Reto Da Forno's avatar
Reto Da Forno committed
465
466
467
468
469
470
                elif adaptTypesFk and platf:
                    # Cycle through the adaptors which are attached to the observer and try to find one that can be used with the requested platform:
                    adaptFound = False
                    for adapt in adaptTypesFk:
                        # Only check for entries which are not null:
                        if adapt:
471
472
                            cur.execute(sql_platf %(adapt, platf))
                            rs = cur.fetchone()
Reto Da Forno's avatar
Reto Da Forno committed
473
474
475
476
477
                            if (rs[0] > 0):
                                adaptFound = True
                                break
                    if not adaptFound:
                        if not quiet:
478
                            print(("Element targetConf: Observer ID %s has currently no target adapter for %s installed." %(obsid, platf)))
Reto Da Forno's avatar
Reto Da Forno committed
479
480
                        errcnt = errcnt + 1
                if platf is not None:
481
482
                    cur.execute(sql_cores %(platf))
                    core_info = cur.fetchall()
Reto Da Forno's avatar
Reto Da Forno committed
483
484
485
486
487
                    all_cores = [row[0] for row in core_info]
                    required_cores = [row[0] for row in [row for row in core_info if row[1]==0]]
                    provided_cores = list(obsiddict[obsid].keys())
                    if not set(required_cores).issubset(set(provided_cores)):
                        if not quiet:
488
                            print(("Element targetConf: Not enough target images provided for Observer ID %s. Platform %s requires images for cores %s." %(obsid, platf, ','.join(map(str,required_cores)))))
Reto Da Forno's avatar
Reto Da Forno committed
489
490
491
                        errcnt = errcnt + 1
                    if not set(provided_cores).issubset(set(all_cores)):
                        if not quiet:
492
                            print(("Element targetConf: Excess target images specified on Observer ID %s. Platform %s requires images for cores %s." %(obsid, platf, ','.join(map(str,required_cores)))))
Reto Da Forno's avatar
Reto Da Forno committed
493
494
495
496
497
498
499
500
501
502
503
                        errcnt = errcnt + 1
    
    #===========================================================================
    # If there are still no errors, do additional test on the remaining elements
    #===========================================================================
    if errcnt == 0:
        # serialConf additional validation --------------------------------------
        #    * observer ids need to have a targetConf associated and must be unique
        #    * check port depending on platform 
        
        # Check observer ids:
504
        (ids, duplicates, allInList) = check_obsids(tree, '//d:serialConf/d:obsIds', ns, obsidlist)
Reto Da Forno's avatar
Reto Da Forno committed
505
506
        if duplicates:
            if not quiet:
507
                print("Element serialConf: Some observer IDs have been used more than once.")
Reto Da Forno's avatar
Reto Da Forno committed
508
509
510
            errcnt = errcnt + 1
        if not allInList:
            if not quiet:
511
                print("Element serialConf: Some observer IDs have been used but do not have a targetConf element associated with them.")
Reto Da Forno's avatar
Reto Da Forno committed
512
513
514
515
516
            errcnt = errcnt + 1
        
        # gpioTracingConf additional validation ---------------------------------------
        #    * observer ids need to have a targetConf associated and must be unique
        #    * Every (pin, edge) combination can only be used once.
517

Reto Da Forno's avatar
Reto Da Forno committed
518
        # Check observer ids:
519
        (ids, duplicates, allInList) = check_obsids(tree, '//d:gpioTracingConf/d:obsIds', ns, obsidlist)
Reto Da Forno's avatar
Reto Da Forno committed
520
521
        if duplicates:
            if not quiet:
522
                print("Element gpioTracingConf: Some observer IDs have been used more than once.")
Reto Da Forno's avatar
Reto Da Forno committed
523
524
525
            errcnt = errcnt + 1
        if not allInList:
            if not quiet:
526
                print("Element gpioTracingConf: Some observer IDs have been used but do not have a) targetConf element associated with them.")
Reto Da Forno's avatar
Reto Da Forno committed
527
528
529
530
531
532
533
534
535
536
537
538
            errcnt = errcnt + 1
        # Check (pin, edge) combinations:
        gpiomonconfs = tree.xpath('//d:gpioTracingConf', namespaces=ns)
        for gpiomonconf in gpiomonconfs:
            combList = []
            pinconfs = gpiomonconf.xpath('d:pinConf', namespaces=ns)
            for pinconf in pinconfs:
                pin  = pinconf.xpath('d:pin', namespaces=ns)[0].text
                edge = pinconf.xpath('d:edge', namespaces=ns)[0].text
                combList.append((pin, edge))
            if (len(combList) != len(set(combList))):
                if not quiet:
539
                    print(("Line %d: element gpioTracingConf: Every (pin, edge) combination can only be used once per observer configuration." %(gpiomonconf.sourceline)))
Reto Da Forno's avatar
Reto Da Forno committed
540
                errcnt = errcnt + 1
541
542
        
        
Reto Da Forno's avatar
Reto Da Forno committed
543
544
545
546
547
548
        # gpioActuationConf additional validation ---------------------------
        #    * observer ids need to have a targetConf associated and must be unique
        #    * relative timing commands cannot be after the test end
        #    * absolute timing commands need to be between test start and end and are not allowed for ASAP test scheduling
        
        # Check observer ids:
549
        (ids, duplicates, allInList) = check_obsids(tree, '//d:gpioActuationConf/d:obsIds', ns, obsidlist)
Reto Da Forno's avatar
Reto Da Forno committed
550
551
        if duplicates:
            if not quiet:
552
                print("Element gpioActuationConf: Some observer IDs have been used more than once.")
Reto Da Forno's avatar
Reto Da Forno committed
553
554
555
            errcnt = errcnt + 1
        if not allInList:
            if not quiet:
556
                print("Element gpioActuationConf: Some observer IDs have been used but do not have a targetConf element associated with them.")
Reto Da Forno's avatar
Reto Da Forno committed
557
558
559
560
561
562
            errcnt = errcnt + 1
        # Check relative timings:
        rs = tree.xpath('//d:gpioActuationConf/d:pinConf/d:relativeTime/d:offsetSecs', namespaces=ns)
        for elem in rs:
            if (int(elem.text) > testDuration):
                if not quiet:
563
                    print(("Line %d: element offsetSecs: The offset is bigger than the test duration, thus the action will never take place." %(elem.sourceline)))
Reto Da Forno's avatar
Reto Da Forno committed
564
565
566
567
568
569
                errcnt = errcnt + 1
        # Check absolute timings:
        rs = tree.xpath('//d:gpioActuationConf/d:pinConf/d:absoluteTime/d:absoluteDateTime', namespaces=ns)
        for elem in rs:
            if sched_asap:
                if not quiet:
570
                    print(("Line %d: element absoluteDateTime: For test scheduling method ASAP, only relative timed actions are allowed." %(elem.sourceline)))
Reto Da Forno's avatar
Reto Da Forno committed
571
572
                errcnt = errcnt + 1
            else:
573
                eventTime = flocklab.get_xml_timestamp(elem.text)
Reto Da Forno's avatar
Reto Da Forno committed
574
575
                if (eventTime > testEnd):
                    if not quiet:
576
                        print(("Line %d: element absoluteDateTime: The action is scheduled after the test ends, thus the action will never take place." %(elem.sourceline)))
Reto Da Forno's avatar
Reto Da Forno committed
577
578
579
                    errcnt = errcnt + 1
                elif (eventTime < testStart):
                    if not quiet:
580
                        print(("Line %d: element absoluteDateTime: The action is scheduled before the test starts, thus the action will never take place." %(elem.sourceline)))
Reto Da Forno's avatar
Reto Da Forno committed
581
582
                    errcnt = errcnt + 1
        
583

Reto Da Forno's avatar
Reto Da Forno committed
584
585
586
587
        # powerProfilingConf additional validation -----------------------------------------
        #    * observer ids need to have a targetConf associated and must be unique
        #    * relative timing commands cannot be after the test end
        #    * absolute timing commands need to be between test start and end and are not allowed for ASAP test scheduling
588

Reto Da Forno's avatar
Reto Da Forno committed
589
        # Check observer ids:
590
        (ids, duplicates, allInList) = check_obsids(tree, '//d:powerProfilingConf/d:obsIds', ns, obsidlist)
Reto Da Forno's avatar
Reto Da Forno committed
591
592
        if duplicates:
            if not quiet:
593
                print("Element powerProfilingConf: Some observer IDs have been used more than once.")
Reto Da Forno's avatar
Reto Da Forno committed
594
595
596
            errcnt = errcnt + 1
        if not allInList:
            if not quiet:
597
                print("Element powerProfilingConf: Some observer IDs have been used but do not have a targetConf element associated with them.")
Reto Da Forno's avatar
Reto Da Forno committed
598
            errcnt = errcnt + 1
Reto Da Forno's avatar
Reto Da Forno committed
599
600
601
602
603
        # Check simple offset tag
        rs = tree.xpath('//d:powerProfilingConf/d:profConf/d:offset', namespaces=ns)
        for elem in rs:
            ppStart = int(elem.text)
            elem2 = elem.getparent().find('d:durationMillisecs', namespaces=ns)
Reto Da Forno's avatar
Reto Da Forno committed
604
            if elem2 is not None:
Reto Da Forno's avatar
Reto Da Forno committed
605
606
607
608
609
610
611
612
613
614
615
616
                ppDuration = int(elem2.text) / 1000
            else:
                elem2 = elem.getparent().find('d:duration', namespaces=ns)
                ppDuration = int(elem2.text)
            if (ppStart > testDuration):
                if not quiet:
                    print(("Line %d: element offset: The offset is bigger than the test duration, thus the action will never take place." % (elem.sourceline)))
                errcnt = errcnt + 1
            elif (ppStart + ppDuration > testDuration):
                if not quiet:
                    print(("Line %d: element duration/durationMillisecs: Profiling lasts longer than test." % (elem2.sourceline)))
                errcnt = errcnt + 1
Reto Da Forno's avatar
Reto Da Forno committed
617
618
619
620
621
622
623
624
625
        # Check relative timings:
        rs = tree.xpath('//d:powerProfilingConf/d:profConf/d:relativeTime/d:offsetSecs', namespaces=ns)
        for elem in rs:
            ppMicroSecs = elem.getparent().find('d:offsetMicrosecs', namespaces=ns)
            if ppMicroSecs is not None:
                ppStart = float(ppMicroSecs.text) / 1000000 + int(elem.text)
            else:
                ppStart = int(elem.text)
            elem2 = elem.getparent().getparent().find('d:durationMillisecs', namespaces=ns)
Reto Da Forno's avatar
Reto Da Forno committed
626
            if elem2 is not None:
Reto Da Forno's avatar
Reto Da Forno committed
627
628
629
630
                ppDuration = int(elem2.text) / 1000
            else:
                elem2 = elem.getparent().getparent().find('d:duration', namespaces=ns)
                ppDuration = int(elem2.text)
Reto Da Forno's avatar
Reto Da Forno committed
631
632
            if (ppStart > testDuration):
                if not quiet:
Reto Da Forno's avatar
Reto Da Forno committed
633
                    print(("Line %d: element offsetSecs: The offset is bigger than the test duration, thus the action will never take place." % (elem.sourceline)))
Reto Da Forno's avatar
Reto Da Forno committed
634
                errcnt = errcnt + 1
Reto Da Forno's avatar
Reto Da Forno committed
635
            elif (ppStart + ppDuration > testDuration):
Reto Da Forno's avatar
Reto Da Forno committed
636
                if not quiet:
Reto Da Forno's avatar
Reto Da Forno committed
637
                    print(("Line %d: element duration/durationMillisecs: Profiling lasts longer than test." % (elem2.sourceline)))
Reto Da Forno's avatar
Reto Da Forno committed
638
639
640
641
642
643
                errcnt = errcnt + 1
        # Check absolute timings:
        rs = tree.xpath('//d:powerProfilingConf/d:profConf/d:absoluteTime/d:absoluteDateTime', namespaces=ns)
        for elem in rs:
            if sched_asap:
                if not quiet:
644
                    print(("Line %d: element absoluteDateTime: For test scheduling method ASAP, only relative timed actions are allowed." %(elem.sourceline)))
Reto Da Forno's avatar
Reto Da Forno committed
645
646
647
                errcnt = errcnt + 1
            else:
                ppMicroSecs = elem.getparent().find('d:absoluteMicrosecs', namespaces=ns)
648
                eventTime = flocklab.get_xml_timestamp(elem.text)
Reto Da Forno's avatar
Reto Da Forno committed
649
650
651
652
653
                if ppMicroSecs is not None:
                    ppStart = float(ppMicroSecs.text) / 1000000 + eventTime
                else:
                    ppStart = eventTime
                elem2 = elem.getparent().getparent().find('d:durationMillisecs', namespaces=ns)
Reto Da Forno's avatar
Reto Da Forno committed
654
                if elem2 is not None:
Reto Da Forno's avatar
Reto Da Forno committed
655
656
657
658
                    ppDuration = int(elem2.text) / 1000
                else:
                    elem2 = elem.getparent().getparent().find('d:duration', namespaces=ns)
                    ppDuration = int(elem2.text)
Reto Da Forno's avatar
Reto Da Forno committed
659
660
                if (ppStart > testEnd):
                    if not quiet:
661
                        print(("Line %d: element absoluteDateTime: The action is scheduled after the test ends, thus the action will never take place." %(elem.sourceline)))
Reto Da Forno's avatar
Reto Da Forno committed
662
663
664
                    errcnt = errcnt + 1
                elif (ppStart < testStart):
                    if not quiet:
665
                        print(("Line %d: element absoluteDateTime: The action is scheduled before the test starts, thus the action will never take place." %(elem.sourceline)))
Reto Da Forno's avatar
Reto Da Forno committed
666
                    errcnt = errcnt + 1
Reto Da Forno's avatar
Reto Da Forno committed
667
                elif (ppStart + ppDuration > testEnd):
Reto Da Forno's avatar
Reto Da Forno committed
668
                    if not quiet:
Reto Da Forno's avatar
Reto Da Forno committed
669
                        print(("Line %d: element duration/durationMillisecs: Profiling lasts longer than test." % (elem2.sourceline)))
Reto Da Forno's avatar
Reto Da Forno committed
670
671
672
673
674
                    errcnt = errcnt + 1
    
    #===========================================================================
    # All additional tests finished. Clean up and exit.
    #===========================================================================
675
676
    if cn.open:
        cn.close()
Reto Da Forno's avatar
Reto Da Forno committed
677
678
679
680
681
682
683
684
    
    # If there is a temp XML file, delete it:
    if testid:
        os.remove(xmlpath)
    
    logger.debug("Validation finished (%u errors)." % errcnt)
    
    if errcnt == 0:
Reto Da Forno's avatar
Reto Da Forno committed
685
        ret = flocklab.SUCCESS
Reto Da Forno's avatar
Reto Da Forno committed
686
    else:
687
        err_str = "Number of errors: %d. It is possible that there are more errors which could not be detected due to dependencies from above listed errors."%errcnt
Reto Da Forno's avatar
Reto Da Forno committed
688
689
690
691
692
        logger.debug(err_str)
        if not quiet:
            print(err_str)
        ret = errno.EBADMSG
    sys.exit(ret)
693
694
695
### END main()

if __name__ == "__main__":
Reto Da Forno's avatar
Reto Da Forno committed
696
697
698
699
700
    try:
        main(sys.argv[1:])
    except Exception:
        print("testconfig validator encountered an error: %s: %s\n%s" % (str(sys.exc_info()[0]), str(sys.exc_info()[1]), traceback.format_exc()))
        sys.exit(errno.EBADMSG)