diff -r 82f11024044a -r 932c358ece3e Orb/python/orb/guidiser.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Orb/python/orb/guidiser.py Fri Apr 23 20:45:58 2010 +0100 @@ -0,0 +1,613 @@ +# Copyright (c) 2007-2010 Nokia Corporation and/or its subsidiary(-ies) All rights reserved. +# This component and the accompanying materials are made available under the terms of the License +# "Eclipse Public License v1.0" which accompanies this distribution, +# and is available at the URL "http://www.eclipse.org/legal/epl-v10.html". +# +# Initial Contributors: +# Nokia Corporation - initial contribution. +# +# Contributors: +# +# Description: +# +from __future__ import with_statement +import unittest +import uuid +import os +import stat +import sys +import shutil +import xml +import logging +from optparse import OptionParser, check_choice +from xml.etree import ElementTree as etree +from cStringIO import StringIO +from lib import scan, xml_decl, doctype_identifier, XmlParser +from doxyidredirect import DoxyIdRedirect, ExceptionDoxyIdRedirectLookup + + +__version__ = "0.1" + +class Guidiser(object): + """ + A simple class that parses an xml file and converts the values of all + id, href and keyref attributes to a 'GUID'. + + >>> guid = Guidiser() + >>> root = guid.guidise(StringIO(cxxclass)) + >>> oldroot = etree.parse(StringIO(cxxclass)).getroot() + >>> oldroot.attrib['id'] + 'CP_class' + >>> root.attrib['id'] + 'GUID-25825EC4-341F-3EA4-94AA-7DCE380E6D2E' + """ + # Publishing targets + PT_MODE = 0 + PT_DITAOT = 1 + PUBLISHING_TARGETS = (PT_MODE, PT_DITAOT) + + def __init__(self, namespace='www.nokia.com', publishing_target=0, xmlparser=XmlParser(), doxyidredirect=DoxyIdRedirect(None)): + self.namespace = self._get_namespace(namespace) + self.set_publishing_target(publishing_target) + self.xmlparser = xmlparser + self.doxyidredirect = doxyidredirect + + def set_publishing_target(self, target): + if not target in self.PUBLISHING_TARGETS: + raise Exception('Invalid Publishing Target \"%s\"' % target) + self._publishing_target = target + + def get_publishing_target(self): + return self._publishing_target + + def _get_namespace(self, namespace, LEN_BYTES=16): + if len(namespace) < LEN_BYTES: + namespace = namespace + (' ' * (LEN_BYTES - len(namespace))) + return uuid.UUID(bytes=namespace[:LEN_BYTES]) + + def _get_guid(self, fqn): + return ('GUID-%s' % (uuid.uuid3(self.namespace, fqn))).upper() + + def _guidise_href(self, href, tag): + if tag == "xref": + return self._guidise_xref_href(href) + else: + # Tag is a topicref or topicref descended element + return self._guidise_topicref_href(href) + + def _guidise_topicref_href(self, href): + # Guidise an href that points to a ditamap + # NOTE: the id of the map is assumed to be the same as the filename + # (minus the ".ditamap" extension) + if href.endswith(".ditamap"): + guid = self._get_guid(href[:-len(".ditamap")]) + if self.get_publishing_target() == self.PT_DITAOT: + guid += ".ditamap" + return guid + + # Guidise an href that points to a topic + # NOTE: Doxygen currently outputs "filepath#topicid" for topicref hrefs + # the "#topicid" is redundant (as topicrefs can't reference below the topic level) + # so will probably be removed from doxygen output at some point. + filename = href.split('#')[0] + id = os.path.splitext(filename)[0] + fqn = None + if not(id.lower() in ("test", "deprecated", "todo") or id.lower().find("namespace_") != -1): + try: + filename, fqn = self.doxyidredirect.lookupId(id) + except ExceptionDoxyIdRedirectLookup, err: + logging.error("Could not lookup Fully Qualified APIName for id '%s' in href '%s'" % (id, href)) + #if the id was not found just guidise the id + #this is just to make the id unique for mode + guid = self._get_guid(fqn) if fqn else self._get_guid(id) + if self.get_publishing_target() == self.PT_DITAOT: + guid+=".xml" + return guid + + def _guidise_xref_href(self, href): + # Don't guidise references without hashes. Assume they are filepaths + # to files other than ditatopics + if href.find('#') == -1: + return href + + # Doxygen currently outputs hrefs in the format autolink_8cpp.xml#autolink_8cpp_1ae0e289308b6d2cbb5c86e753741981dc + # The right side of the # is not enough to extract the fully qualified name of the function because it is md5ed + # Send the right side to doxyidredirect to get the fqn of the function + filename, id = href.split('#') + fqn = None + if not(id.lower() in ("test", "deprecated", "todo") or id.lower().find("namespace_") != -1): + try: + fqn = self.doxyidredirect.lookupId(id)[1] + except ExceptionDoxyIdRedirectLookup, err: + logging.error("No API name for element id %s, guidising id instead" % id) + + guid = self._get_guid(fqn) if fqn else self._get_guid(id) + basename, ext = os.path.splitext(filename) + try: + base_guid = self._get_guid(self.doxyidredirect.lookupId(basename)[1]) + except ExceptionDoxyIdRedirectLookup, e: + base_guid = self._get_guid(basename) + + if self.get_publishing_target() == self.PT_DITAOT: + return base_guid + ext + "#" + guid + else: + return guid + + def _guidise_id(self, id): + try: + filename, fqn = self.doxyidredirect.lookupId(id) + return self._get_guid(fqn) + except ExceptionDoxyIdRedirectLookup, err: + logging.debug("Didn't find a Fully Qualified APIName for id '%s'" % id) + return self._get_guid(id) + + def guidise(self, xmlfile): + #WORKAROUND: ElementTree provides no function to set prefixes and makes up its own if they are not set (ns0, ns1, ns2) + etree._namespace_map["http://dita.oasis-open.org/architecture/2005/"] = 'ditaarch' + try: + root = etree.parse(xmlfile).getroot() + except xml.parsers.expat.ExpatError, e: + logging.error("%s could not be parsed: %s\n" % (xmlfile, str(e))) + return None + for child in root.getiterator(): + for key in [key for key in ('id', 'href', 'keyref') if key in child.attrib]: + if key == 'id': + child.attrib['id'] = self._guidise_id(child.attrib['id']) + elif key == 'href': + if 'format' in child.attrib and child.attrib['format'] == 'html': + continue + else: + base_dir = os.path.dirname(xmlfile) if isinstance(xmlfile, str) else "" + child.attrib['href'] = self._guidise_href(child.attrib['href'], child.tag) + elif key == 'keyref': + child.attrib['keyref'] = self._get_guid(child.attrib['keyref']) + + return root + + +def updatefiles(xmldir, publishing_target="ditaot"): + publishing_target = Guidiser.PT_MODE if (publishing_target == "mode") else Guidiser.PT_DITAOT + guidiser = Guidiser(publishing_target=publishing_target, doxyidredirect=DoxyIdRedirect(xmldir)) + for filepath in scan(xmldir): + logging.debug('Guidising file \"%s\"' % filepath) + root = guidiser.guidise(filepath) + if root is not None: + try: + os.chmod(filepath, stat.S_IWRITE) + except Exception, e: + logging.error("Could not make file \"%s\" writable, error was \"%s\"" % (filepath, e)) + continue + with open(filepath, 'w') as f: + f.write(xml_decl()+'\n') + try: + doc_id = doctype_identifier(root.tag) + except Exception, e: + logging.error("Could not write doctype identifier for file \"%s\", error was \"%s\"" + %(filepath, e)) + else: + f.write(doc_id+'\n') + f.write(etree.tostring(root)) + f.close() + +def main(): + usage = "usage: %prog [options] " + parser = OptionParser(usage, version='%prog ' + __version__) + parser.add_option("-p", dest="publishing_target", type="choice", choices=["mode", "ditaot"], default="mode", + help="Publishing Target: mode|ditaot, [default: %default]") + parser.add_option("-l", "--loglevel", type="int", default=30, help="Log Level (debug=10, info=20, warning=30, [error=40], critical=50)") + (options, args) = parser.parse_args() + if len(args) < 1: + parser.print_help() + parser.error("Please supply the path to the XML content") + if options.loglevel: + logging.basicConfig(level=options.loglevel) + updatefiles(args[0], options.publishing_target) + + +if __name__ == '__main__': + sys.exit(main()) + + +###################################### +# Test code +###################################### + +class StubDoxyIdRedirect(object): + def __init__(self, theDir): + self.dict = {'struct_e_sock_1_1_t_addr_update':('struct_e_sock_1_1_t_addr_update.xml', 'ESock::TAddrUpdate'), + 'class_c_active_scheduler_1_1_t_cleanup_bundle':('class_c_active_scheduler_1_1_t_cleanup_bundle.xml', 'CActiveScheduler::TCleanupBundle'), + 'class_test':('class_test.xml', 'Test'), + 'class_test_1a99f2bbfac6c95612322b0f10e607ebe5':('cxxclass.xml', 'Test')} + + def lookupId(self, doxy_id): + try: + filename, fqn = self.dict[doxy_id] + return (filename, fqn) + except Exception, e: + raise ExceptionDoxyIdRedirectLookup("StubException: %s" % e) + + +class TestGuidiser(unittest.TestCase): + def setUp(self): + self.guidiser = Guidiser(publishing_target=Guidiser.PT_MODE, doxyidredirect=StubDoxyIdRedirect('adir')) + self.test_dir = "guidiser_test_dir" + + def _create_test_data(self): + f = open("struct_e_sock_1_1_t_addr_update.xml", "w") + f.write(struct_e_sock_1_1_t_addr_update) + f.close() + os.mkdir(self.test_dir) + f = open(os.path.join(self.test_dir, "struct_e_sock_1_1_t_addr_update.xml"), "w") + f.write(struct_e_sock_1_1_t_addr_update) + f.close() + + def _cleanup_test_data(self): + os.remove("struct_e_sock_1_1_t_addr_update.xml") + shutil.rmtree(self.test_dir) + + def test_i_can_get_and_set_a_PT(self): + self.assertEqual(self.guidiser.get_publishing_target(), Guidiser.PT_MODE) + self.guidiser.set_publishing_target(Guidiser.PT_DITAOT) + self.assertEqual(self.guidiser.get_publishing_target(), Guidiser.PT_DITAOT) + + def test_i_raise_an_exception_when_trying_to_set_an_invalid_PT(self): + self.assertRaises(Exception, self.guidiser.set_publishing_target, 2) + + def test_i_update_root_elements_id(self): + root = self.guidiser.guidise(StringIO(cxxclass)) + self.assertEqual(root.attrib['id'], "GUID-56866D87-2CE9-31EA-8FA7-F4275FDBCB93") + + def test_i_continue_if_passed_an_invalid_file(self): + try: + self.guidiser.guidise(StringIO("")) + except Exception: + self.fail("I shouldnt have raised an exception") + + def _test_keys_were_converted(self, key): + root = self.guidiser.guidise(StringIO(cxxclass)) + for child in root.getiterator(): + if key in child.attrib: + self.assertTrue(child.attrib[key].startswith('GUID')) + + def test_i_update_a_subelements_id(self): + self._test_keys_were_converted('id') + + def test_i_update_all_hrefs_with_a_guid(self): + self._test_keys_were_converted('href') + + def test_i_update_all_keyrefs_with_a_guid(self): + self._test_keys_were_converted('keyref') + + def test_based_fqn_and_one_param(self): + self.assertTrue(self.guidiser._get_guid("RConnection::EnumerateConnections(TUint&)") == + "GUID-18F9018F-78DE-3A7E-8363-B7CB101E7A99" + ) + + def test_based_fqn_and_muiltiple_params(self): + self.assertTrue(self.guidiser._get_guid("RConnection::ProgressNotification(TSubConnectionUniqueId, TNifProgressBuf&, TRequestStatus&, TUint)") == + "GUID-6E7005CF-4D8E-31CE-BAEA-21965ACC9C17" + ) + + def test_based_fqn_and_muiltiple_params_ones_a_default(self): + self.assertTrue(self.guidiser._get_guid("RConnection::Open(RSocketServ& aSocketServer, TUint aConnectionType = KConnectionTypeDefault)") == + "GUID-CE8F3FE7-14F2-3FB6-B04C-8596B5F80DFC" + ) + + def test_based_fqn_and_muiltiple_params_ones_templated(self): + self.assertTrue(self.guidiser._get_guid("RConnection::DataReceivedNotificationRequest(TSubConnectionUniqueId, TUint, TPckg&, TRequestStatus&)") == + "GUID-9E056551-22C2-3F85-8E3D-C11FA3B46F07" + ) + + def test_based_toplevel_class(self): + self.assertTrue(self.guidiser._get_guid("RConnection") == + "GUID-BED8A733-2ED7-31AD-A911-C1F4707C67FD" + ) + + def test_target_id(self): + self.assertTrue(self.guidiser._get_guid("ESock::TAddrUpdate") == + "GUID-E72084E6-C1CE-3388-93F7-5B7A3F506C3B" + ) + + def test_topicref_href_to_topic_for_mode(self): + self.assertEquals(self.guidiser._guidise_href("struct_e_sock_1_1_t_addr_update.xml#struct_e_sock_1_1_t_addr_update", "topicref"), + "GUID-E72084E6-C1CE-3388-93F7-5B7A3F506C3B" + ) + + def test_topicref_href_to_topic_for_ditaot(self): + self.guidiser.set_publishing_target(Guidiser.PT_DITAOT) + self._create_test_data() + try: + self.assertEquals(self.guidiser._guidise_href("struct_e_sock_1_1_t_addr_update.xml#struct_e_sock_1_1_t_addr_update", "topicref"), + "GUID-E72084E6-C1CE-3388-93F7-5B7A3F506C3B.xml") + finally: + self._cleanup_test_data() + + + def test_topicref_href_to_map_for_mode(self): + self.assertEquals(self.guidiser._guidise_href("ziplib.ditamap", "topicref"), + "GUID-7C7A889C-AE2B-31FC-A5DA-A87019E1251D" + ) + + def test_topicref_href_to_map_for_ditaot(self): + self.guidiser.set_publishing_target(Guidiser.PT_DITAOT) + self.assertEquals(self.guidiser._guidise_href("ziplib.ditamap", "topicref"), + "GUID-7C7A889C-AE2B-31FC-A5DA-A87019E1251D.ditamap" + ) + + def test_xref_href_to_topic_in_same_file_for_mode(self): + self.assertEquals(self.guidiser._guidise_href("struct_e_sock_1_1_t_addr_update.xml#struct_e_sock_1_1_t_addr_update", "xref"), + "GUID-E72084E6-C1CE-3388-93F7-5B7A3F506C3B" + ) + + def test_xref_href_to_topic_in_same_file_for_ditaot(self): + self.guidiser.set_publishing_target(Guidiser.PT_DITAOT) + self.assertEquals(self.guidiser._guidise_href("struct_e_sock_1_1_t_addr_update.xml#struct_e_sock_1_1_t_addr_update", "xref"), + "GUID-E72084E6-C1CE-3388-93F7-5B7A3F506C3B.xml#GUID-E72084E6-C1CE-3388-93F7-5B7A3F506C3B" + ) + + def test_xref_href_to_some_other_file_on_file_system(self): + self.guidiser.set_publishing_target(Guidiser.PT_DITAOT) + self.assertEquals(self.guidiser._guidise_href("../../documentation/RFCs/rfc3580.txt", "xref"), + "../../documentation/RFCs/rfc3580.txt" + ) + + def test_i_guidise_the_id_of_a_fully_qualified_apiname(self): + self.assertEquals(self.guidiser._guidise_id("struct_e_sock_1_1_t_addr_update"), + "GUID-E72084E6-C1CE-3388-93F7-5B7A3F506C3B" + ) + + def test_id_guidise_the_id_something_that_is_not_a_fully_qualified_apiname(self): + self.assertEquals(self.guidiser._guidise_id("commsdataobjects"), + "GUID-2F2463E0-6C84-3FAB-8B60-57E57315FDEB" + ) + + def test_i_preserve_namespaces(self): + xml_in = """""" + xml_expected = """""" + root = self.guidiser.guidise(StringIO(xml_in)) + self.assertEqual(etree.tostring(root), xml_expected) + +class Testupdate_files(unittest.TestCase): + + def setUp(self): + self.test_dir = "guidisertestdata" + + def tearDown(self): + shutil.rmtree(self.test_dir) + + def test_i_can_update_a_file_on_the_file_sys(self): + def reference_file_handle(mode): + return open(os.path.join(self.test_dir, "reference.dita"), mode) + os.mkdir(self.test_dir) + f = reference_file_handle("w") + f.write(filesys_cxxclass) + f.close() + updatefiles(self.test_dir) + self.assertEquals(reference_file_handle("r").read(), filesys_cxxclass_guidised) + +struct_e_sock_1_1_t_addr_update = """ + + + CoordStruct + + + + + + + + + + + + + +

A coordinate pair.

+ + + + x + + + + + float + CoordStruct + float x + CoordStruct::x + + + + + + +

The x coordinate

+ + + + + y + + + + + float + CoordStruct + float y + CoordStruct::y + + + + + + +

The y coordinate

+ + + +""" + +filesys_cxxclass = """ + + + CActiveScheduler::TCleanupBundle + + + + + + + + + + + + + + + + iCleanupPtr + + + + + + CCleanup * + CActiveScheduler::TCleanupBundle + CCleanup * iCleanupPtr + CActiveScheduler::TCleanupBundle::iCleanupPtr + + + + + + + + + + iDummyInt + + + + + + TInt + + CActiveScheduler::TCleanupBundle + TInt iDummyInt + CActiveScheduler::TCleanupBundle::iDummyInt + + + + + + + + +""" + +filesys_cxxclass_guidised = """ + + + CActiveScheduler::TCleanupBundle + + + + + + + + + + + + + + + + iCleanupPtr + + + + + + CCleanup * + CActiveScheduler::TCleanupBundle + CCleanup * iCleanupPtr + CActiveScheduler::TCleanupBundle::iCleanupPtr + + + + + + + + + + iDummyInt + + + + + + TInt + + CActiveScheduler::TCleanupBundle + TInt iDummyInt + CActiveScheduler::TCleanupBundle::iDummyInt + + + + + + + + +""" + +cxxclass = """ + + + Test + + + + + + + + + + + + + +

Points to function Test()

+ + + + Test + + + + + + + Test + Test() + Test::Test() + + + + + + + + + + +

details.

+ + + +""" \ No newline at end of file