CDash:Dart2AndCDashSubmission

From KitwarePublic
Revision as of 23:40, 29 March 2008 by TrevK (talk | contribs)
Jump to navigationJump to search

Setting up dual Dart2 and CDash Submissions

If you are moving from Dart2 to CDash you may want to submit to both dashboards in parallel for a while, before switching solely to CDash.

The following shows how to setup CTest to use a trigger script, which in turn does a submit to both Dart2 (via XML-RPC) and CDash (via HTTP).

Note: This method doesn't support submission of Dart2 notes files.

Configure CTest to use a trigger script:

SET (CTEST_DROP_METHOD "http")
SET (CTEST_DROP_SITE "mysite.org")
SET (CTEST_DROP_LOCATION "/cgi-bin/HTTPUploadDartFile.cgi")
SET (CTEST_TRIGGER_SITE "http://${DROP_SITE}/cgi-bin/TriggerSiteDart2AndCDash.cgi")

Both HTTPUploadDartFile.cgi and TriggerSiteDart2AndCDash.cgi have a hard coded path that may require editing: /srv/dart2/incoming

This current implementation only supports a single hard coded project in TriggerSiteDart2AndCDash.cgi. You can of course create a specific trigger site per project to workaround this.

HTTPUploadDartFile.cgi

#!/usr/bin/env python
#
# Script that will upload files to specified directory on the server, where
# triggering script will find it.
#
# Installation:
#    Place this script in your cgi-bin area.
#    Change the variable "incoming_directory" to match your
#        installation.
###

import cgi
import re
import sys
import os
import string

print "Content-type: text/html"
print

done = 0

incoming_directory = '/srv/dart2/incoming'

form = cgi.FieldStorage()

# Debug
#try:
#  tmpfile = open("/tmp/http_upload_log_file.log", "w")
#  tmpfile.write(`form`)
#  tmpfile.close()
#except Exception: pass

FileData = ""
FileName = ""

# process form
if type(form.value) == type("foo"):
 if os.environ.has_key("QUERY_STRING"):
   dt = cgi.parse_qs(os.environ["QUERY_STRING"])
   if dt.has_key("FileName"):
     FileName = dt["FileName"][0]
   FileData = form.value
elif form.has_key("FileName"):
 FileName = form["FileName"].value
 if form.has_key("FileData"):
    FileData = "form"

# verify file specified
if not FileData or not FileName:
  print "Please send file. "
  print "FileName has to contain file name"
  print "FileData has to contain file content"
  print form
  sys.exit(0)

print "Received file: " + FileName

# check if valid file name
filename = re.sub("[/\\|:%%]", "_", FileName)
if re.match(".+___.+___[0-9]{8}-[0-9]{4}-(Experimental|Nightly|Continuous)___XML___(Update|Configure|Build|Test|Coverage|Purify).xml", filename):
  print "Correct file name"
else:
  print "Can only upload files with format:"
  print "<site>___<BuildType>___<TAG>-<TestType>___XML___<File>.xml"
  sys.exit(0)

# get the file data
file_lines = ""
if FileData == "form":
  fileitem = form["FileData"]
  if fileitem.file:
    file_lines = fileitem.file.read()
  else:
    print "Not a file"
else:
  file_lines = FileData

# write to a file on disk
if file_lines:
  file = open(incoming_directory + "/" + filename, "w")
  if not file:
    print "Cannot create file: %s/%s" % ( incoming_directory, filename )
    sys.exit(0)
  file.write(file_lines)
  file.close()
  print "Thank you for the file"
  done = 1

if not done:
  print "Problem summiting data"

TriggerSiteDart2AndCDash.cgi

Note: If the script fails it will create log files in /tmp/tmp*.html.

#!/usr/bin/env python
#
# Script that will resubmit incoming testing results to
# Dart2 (via XMLRPC) and CDash (via HTTP).
#
# This scripts intended use is to allow the transition from
# Dart2 to CDash, by allowing both to be run in parallel.
#
###

import cgi
import cgitb; cgitb.enable(display=0, logdir="/tmp")

import sys
import os
import xmlrpclib
import put

def Trigger(dart2Url, cdashUrl, projectName, dropLocation):
	print "Content-Type: text/html"
	print

	form = cgi.FieldStorage()
	xmlfile = ""
	if form.has_key("xmlfile"):
		xmlfile = form["xmlfile"].value

	if not xmlfile:
		print "No submission file"
		sys.exit(1)

	ipfile = dropLocation + xmlfile

	try:
		dart2Server = xmlrpclib.ServerProxy(dart2Url + projectName + "/Command/")

		try:
			fp = open(ipfile)
			content = fp.read()
			bin = xmlrpclib.Binary(content)
			print "Server responded: [%s]" % dart2Server.Submit.put(bin)
			fp.close()
		except Exception, v:
			print "ERROR", v
			print "Unexpected error:", sys.exc_info()
	except:
		print "Problem submitting XML-RPC for the file: %s" % xmlfile

	try:
		cdashServer = cdashUrl + "CDash/submit.php?project=" + projectName

		f = open(ipfile, 'rb')
		put.putfile(f, cdashServer)
		f.close()
	except:
		print "Problem submitting HTTP for the file: %s" % xmlfile

	os.unlink(ipfile)


if __name__ == "__main__":
	# Ensure all paths have a trailing slash
	# Dart2 URL, CDash URL, project, incoming directory
	Trigger("http://localhost:8081/", "http://localhost/", "MyProject", "/srv/dart2/incoming/")

put.py

Note: This version of put requires Apache2 (for chunking).

#!/usr/bin/env python
"""
put.py - Python HTTP PUT Client
Author: Sean B. Palmer, inamidst.com

Basic API usage, once with optional auth:

import put
put.putname('test.txt', 'http://example.org/test')

f = open('test.txt', 'rb')
put.putfile(f, 'http://example.org/test')
f.close()

bytes = open('test.txt', 'rb').read()
auth = {'username': 'myuser', 'password': 'mypass'}
put.put(bytes, 'http://example.org/test', **auth)
"""

import sys, httplib, urlparse
from optparse import OptionParser

# True by default when running as a script
# Otherwise, we turn the noise off...
verbose = True

def barf(msg):
   print >> sys.stderr, "Error! %s" % msg
   sys.exit(1)

if sys.version_info < (2, 4):
   barf("Requires Python 2.4+")

def parseuri(uri):
   """Parse URI, return (host, port, path) tuple.

   >>> parseuri('http://example.org/testing?somequery#frag')
   ('example.org', 80, '/testing?somequery')
   >>> parseuri('http://example.net:8080/test.html')
   ('example.net', 8080, '/test.html')
   """

   scheme, netplace, path, query, fragid = urlparse.urlsplit(uri)

   if ':' in netplace:
      host, port = netplace.split(':', 2)
      port = int(port)
   else: host, port = netplace, 80

   if query: path += '?' + query

   return host, port, path

def putfile(f, uri, username=None, password=None):
   """HTTP PUT the file f to uri, with optional auth data."""
   host, port, path = parseuri(uri)

   redirect = set([301, 302, 307])
   authenticate = set([401])
   okay = set([200, 201, 204])

   authorized = False
   authorization = None
   tries = 0

   while True:
      # Attempt to HTTP PUT the data
      h = httplib.HTTPConnection(host, port)

      h.putrequest('PUT', path)

      h.putheader('User-Agent', 'put.py/1.0')
      h.putheader('Connection', 'keep-alive')
      h.putheader('Transfer-Encoding', 'chunked')
      h.putheader('Expect', '100-continue')
      h.putheader('Accept', '*/*')
      if authorization:
         h.putheader('Authorization', authorization)
      h.endheaders()

      # Chunked transfer encoding
      # Cf. 'All HTTP/1.1 applications MUST be able to receive and
      # decode the "chunked" transfer-coding'
      # - http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html
      while True:
         bytes = f.read(2048)
         if not bytes: break
         length = len(bytes)
         h.send('%X\r\n' % length)
         h.send(bytes + '\r\n')
      h.send('0\r\n\r\n')
      f.seek(0);

      resp = h.getresponse()
      status = resp.status # an int

      # Got a response, now decide how to act upon it
      if status in redirect:
         location = resp.getheader('Location')
         uri = urlparse.urljoin(uri, location)
         host, port, path = parseuri(uri)

         # We may have to authenticate again
         if authorization:
            authorization = None

      elif status in authenticate:
         # If we've done this already, break
         if authorization:
            # barf("Going around in authentication circles")

           barf("Authentication failed")

         if not (username and password):
           barf("Need a username and password to authenticate with")

         # Get the scheme: Basic or Digest?
         wwwauth = resp.msg['www-authenticate'] # We may need this again
         wauth = wwwauth.lstrip(' \t') # Hence use wauth not wwwauth here
         wauth = wwwauth.replace('\t', ' ')
         i = wauth.index(' ')
         scheme = wauth[:i].lower()

         if scheme in set(['basic', 'digest','Basic','Digest']):
            if verbose:
               msg = "Performing %s Authentication..." % scheme.capitalize()
         else: barf("Unknown authentication scheme: %s" % scheme)

         if scheme == 'basic' or scheme == 'Basic':
            import base64
            userpass = username + ':' + password
            userpass = base64.encodestring(userpass).strip()
            authorized, authorization = True, 'Basic ' + userpass

         elif scheme == 'digest':
            if verbose:
               msg = "uses fragile, undocumented features in urllib2"

               print >> sys.stderr, "Warning! Digest Auth %s" % msg

            import urllib2 # See warning above

            passwd = type('Password', (object,), {
               'find_user_password': lambda self, *args: (username, password),
               'add_password': lambda self, *args: None
            })()

            req = type('Request', (object,), {
               'get_full_url': lambda self: uri,
               'has_data': lambda self: None,
               'get_method': lambda self: 'PUT',
               'get_selector': lambda self: path
            })()

            # Cf. urllib2.AbstractDigestAuthHandler.retry_http_digest_auth
            auth = urllib2.AbstractDigestAuthHandler(passwd)
            token, challenge = wwwauth.split(' ', 1)
            chal = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge))
            userpass = auth.get_authorization(req, chal)
            authorized, authorization = True, 'Digest ' + userpass

      elif status in okay:
         if (username and password) and (not authorized):
            msg = "Warning! The supplied username and password went unused"
            print msg

         if verbose:
            resultLine = "Success! Resource %s"
            statuses = {200: 'modified', 201: 'created', 204: 'modified'}
            print resultLine % statuses[status]

            statusLine = "Response-Status: %s %s"
            print statusLine % (status, resp.reason)

            body = resp.read(58)
            body = body.rstrip('\r\n')
            body = body.encode('string_escape')

            if len(body) >= 58:
               body = body[:57] + '[...]'

            bodyLine = 'Response-Body: "%s"'
            print bodyLine % body
         break

      # @@ raise PutError, do the catching in main?
      else: barf('Got "%s %s"' % (status, resp.reason))

      tries += 1
      if tries >= 50:
         barf("Too many redirects")

   return status, resp

def putname(fn, uri, username=None, password=None):
   """HTTP PUT the file with filename fn to uri, with optional auth data."""
   auth = {'username': username, 'password': password}

   if fn != '-':
      f = open(fn, 'rb')
      status, resp = putfile(f, uri, **auth)
      f.close()
   else: status, resp = putfile(sys.stdin, uri, **auth)

   return status, resp

def put(s, uri, username=None, password=None):
   """HTTP PUT the string s to uri, with optional auth data."""
   try: from cStringIO import StringIO
   except ImportError:
      from StringIO import StringIO

   f = StringIO(s)
   f.seek(0)
   status, resp = putfile(f, uri, username=username, password=password)
   f.close()

   return status, conn

def main(argv=None):
   usage = ('%prog [options] filename uri\n' +
            'The filename may be - for stdin\n' +
            'Use command line password at your own risk!')

   parser = OptionParser(usage=usage)
   parser.add_option('-u', '--username', help='HTTP Auth username')
   parser.add_option('-p', '--password', help='HTTP Auth password')
   parser.add_option('-q', '--quiet', action='store_true', help="shhh!")
   options, args = parser.parse_args(argv)

   if len(args) != 2:
      parser.error("Requires two arguments, filename and uri")

   fn, uri = args
   if not uri.startswith('http:'):
      parser.error("The uri argument must start with 'http:'")

   if ((options.username and not options.password) or
       (options.password and not options.username)):
      parser.error("Must have both username and password or neither")

   global verbose
   verbose = (not options.quiet)

   auth = {'username': options.username, 'password': options.password}
   putname(fn, uri, **auth)

if __name__ == '__main__':
   main()