1 #! /usr/bin/env python

     2 #######################################################################

     3 # Newslist  $Revision: 66429 $

     4 #

     5 # Syntax:

     6 #    newslist [ -a ]

     7 #

     8 # This is a program to create a directory full of HTML pages

     9 # which between them contain links to all the newsgroups available

    10 # on your server.

    11 #

    12 # The -a option causes a complete list of all groups to be read from

    13 # the server rather than just the ones which have appeared since last

    14 # execution. This recreates the local list from scratch. Use this on

    15 # the first invocation of the program, and from time to time thereafter.

    16 #   When new groups are first created they may appear on your server as

    17 # empty groups. By default, empty groups are ignored by the -a option.

    18 # However, these new groups will not be created again, and so will not

    19 # appear in the server's list of 'new groups' at a later date. Hence it

    20 # won't appear until you do a '-a' after some articles have appeared.

    21 #

    22 # I should really keep a list of ignored empty groups and re-check them

    23 # for articles on every run, but I haven't got around to it yet.

    24 #

    25 # This assumes an NNTP news feed.

    26 #

    27 # Feel free to copy, distribute and modify this code for

    28 # non-commercial use. If you make any useful modifications, let me

    29 # know!

    30 #

    31 # (c) Quentin Stafford-Fraser 1994

    32 # fraser@europarc.xerox.com                     qs101@cl.cam.ac.uk

    33 #                                                                     #

    34 #######################################################################

    35 import sys,nntplib, string, marshal, time, os, posix, string

    36 

    37 #######################################################################

    38 # Check these variables before running!                               #

    39 

    40 # Top directory.

    41 # Filenames which don't start with / are taken as being relative to this.

    42 topdir='/anfs/qsbigdisc/web/html/newspage'

    43 

    44 # The name of your NNTP host

    45 # eg.

    46 #    newshost = 'nntp-serv.cl.cam.ac.uk'

    47 # or use following to get the name from the NNTPSERVER environment

    48 # variable:

    49 #    newshost = posix.environ['NNTPSERVER']

    50 newshost = 'nntp-serv.cl.cam.ac.uk'

    51 

    52 # The filename for a local cache of the newsgroup list

    53 treefile = 'grouptree'

    54 

    55 # The filename for descriptions of newsgroups

    56 # I found a suitable one at ftp.uu.net in /uunet-info/newgroups.gz

    57 # You can set this to '' if you don't wish to use one.

    58 descfile = 'newsgroups'

    59 

    60 # The directory in which HTML pages should be created

    61 # eg.

    62 #   pagedir  = '/usr/local/lib/html/newspage'

    63 #   pagedir  = 'pages'

    64 pagedir  = topdir

    65 

    66 # The html prefix which will refer to this directory

    67 # eg.

    68 #   httppref = '/newspage/',

    69 # or leave blank for relative links between pages: (Recommended)

    70 #   httppref = ''

    71 httppref = ''

    72 

    73 # The name of the 'root' news page in this directory.

    74 # A .html suffix will be added.

    75 rootpage = 'root'

    76 

    77 # Set skipempty to 0 if you wish to see links to empty groups as well.

    78 # Only affects the -a option.

    79 skipempty = 1

    80 

    81 # pagelinkicon can contain html to put an icon after links to

    82 # further pages. This helps to make important links stand out.

    83 # Set to '' if not wanted, or '...' is quite a good one.

    84 pagelinkicon='... <img src="http://pelican.cl.cam.ac.uk/icons/page.xbm"> '

    85 

    86 # ---------------------------------------------------------------------

    87 # Less important personal preferences:

    88 

    89 # Sublistsize controls the maximum number of items the will appear as

    90 # an indented sub-list before the whole thing is moved onto a different

    91 # page. The smaller this is, the more pages you will have, but the

    92 # shorter each will be.

    93 sublistsize = 4

    94 

    95 # That should be all.                                                 #

    96 #######################################################################

    97 

    98 for dir in os.curdir, os.environ['HOME']:

    99     rcfile = os.path.join(dir, '.newslistrc.py')

   100     if os.path.exists(rcfile):

   101         print rcfile

   102         execfile(rcfile)

   103         break

   104 

   105 from nntplib import NNTP

   106 from stat import *

   107 

   108 rcsrev = '$Revision: 66429 $'

   109 rcsrev = string.join(filter(lambda s: '$' not in s, string.split(rcsrev)))

   110 desc = {}

   111 

   112 # Make (possibly) relative filenames into absolute ones

   113 treefile = os.path.join(topdir,treefile)

   114 descfile = os.path.join(topdir,descfile)

   115 page = os.path.join(topdir,pagedir)

   116 

   117 # First the bits for creating trees ---------------------------

   118 

   119 # Addtotree creates/augments a tree from a list of group names

   120 def addtotree(tree, groups):

   121     print 'Updating tree...'

   122     for i in groups:

   123         parts = string.splitfields(i,'.')

   124         makeleaf(tree, parts)

   125 

   126 # Makeleaf makes a leaf and the branch leading to it if necessary

   127 def makeleaf(tree,path):

   128     j = path[0]

   129     l = len(path)

   130 

   131     if not tree.has_key(j):

   132         tree[j] = {}

   133     if l == 1:

   134         tree[j]['.'] = '.'

   135     if l > 1:

   136         makeleaf(tree[j],path[1:])

   137 

   138 # Then the bits for outputting trees as pages ----------------

   139 

   140 # Createpage creates an HTML file named <root>.html containing links

   141 # to those groups beginning with <root>.

   142 

   143 def createpage(root, tree, p):

   144     filename = os.path.join(pagedir,root+'.html')

   145     if root == rootpage:

   146         detail = ''

   147     else:

   148         detail = ' under ' + root

   149     f = open(filename,'w')

   150     # f.write('Content-Type: text/html\n')

   151     f.write('<TITLE>Newsgroups available' + detail + '</TITLE>\n')

   152     f.write('<H1>Newsgroups available' + detail +'</H1>\n')

   153     f.write('<A HREF="'+httppref+rootpage+'.html">Back to top level</A><P>\n')

   154     printtree(f,tree,0,p)

   155     f.write('<I>This page automatically created by \'newslist\' v. '+rcsrev+'.')

   156     f.write(time.ctime(time.time()) + '</I><P>')

   157     f.close()

   158 

   159 # Printtree prints the groups as a bulleted list.  Groups with

   160 # more than <sublistsize> subgroups will be put on a separate page.

   161 # Other sets of subgroups are just indented.

   162 

   163 def printtree(f, tree, indent, p):

   164     global desc

   165     l = len(tree)

   166 

   167     if l > sublistsize and indent>0:

   168         # Create a new page and a link to it

   169         f.write('<LI><B><A HREF="'+httppref+p[1:]+'.html">')

   170         f.write(p[1:]+'.*')

   171         f.write('</A></B>'+pagelinkicon+'\n')

   172         createpage(p[1:], tree, p)

   173         return

   174 

   175     kl = tree.keys()

   176 

   177     if l > 1:

   178         kl.sort()

   179         if indent > 0:

   180             # Create a sub-list

   181             f.write('<LI>'+p[1:]+'\n<UL>')

   182         else:

   183             # Create a main list

   184             f.write('<UL>')

   185         indent = indent + 1

   186 

   187     for i in kl:

   188         if i == '.':

   189             # Output a newsgroup

   190             f.write('<LI><A HREF="news:' + p[1:] + '">'+ p[1:] + '</A> ')

   191             if desc.has_key(p[1:]):

   192                 f.write('     <I>'+desc[p[1:]]+'</I>\n')

   193             else:

   194                 f.write('\n')

   195         else:

   196             # Output a hierarchy

   197             printtree(f,tree[i], indent, p+'.'+i)

   198 

   199     if l > 1:

   200         f.write('\n</UL>')

   201 

   202 # Reading descriptions file ---------------------------------------

   203 

   204 # This returns an array mapping group name to its description

   205 

   206 def readdesc(descfile):

   207     global desc

   208 

   209     desc = {}

   210 

   211     if descfile == '':

   212         return

   213 

   214     try:

   215         d = open(descfile, 'r')

   216         print 'Reading descriptions...'

   217     except (IOError):

   218         print 'Failed to open description file ' + descfile

   219         return

   220     l = d.readline()

   221     while l != '':

   222         bits = string.split(l)

   223         try:

   224             grp = bits[0]

   225             dsc = string.join(bits[1:])

   226             if len(dsc)>1:

   227                 desc[grp] = dsc

   228         except (IndexError):

   229             pass

   230         l = d.readline()

   231 

   232 # Check that ouput directory exists, ------------------------------

   233 # and offer to create it if not

   234 

   235 def checkopdir(pagedir):

   236     if not os.path.isdir(pagedir):

   237         print 'Directory '+pagedir+' does not exist.'

   238         print 'Shall I create it for you? (y/n)'

   239         if sys.stdin.readline()[0] == 'y':

   240             try:

   241                 os.mkdir(pagedir,0777)

   242             except:

   243                 print 'Sorry - failed!'

   244                 sys.exit(1)

   245         else:

   246             print 'OK. Exiting.'

   247             sys.exit(1)

   248 

   249 # Read and write current local tree ----------------------------------

   250 

   251 def readlocallist(treefile):

   252     print 'Reading current local group list...'

   253     tree = {}

   254     try:

   255         treetime = time.localtime(os.stat(treefile)[ST_MTIME])

   256     except:

   257         print '\n*** Failed to open local group cache '+treefile

   258         print 'If this is the first time you have run newslist, then'

   259         print 'use the -a option to create it.'

   260         sys.exit(1)

   261     treedate = '%02d%02d%02d' % (treetime[0] % 100 ,treetime[1], treetime[2])

   262     try:

   263         dump = open(treefile,'r')

   264         tree = marshal.load(dump)

   265         dump.close()

   266     except (IOError):

   267         print 'Cannot open local group list ' + treefile

   268     return (tree, treedate)

   269 

   270 def writelocallist(treefile, tree):

   271     try:

   272         dump = open(treefile,'w')

   273         groups = marshal.dump(tree,dump)

   274         dump.close()

   275         print 'Saved list to '+treefile+'\n'

   276     except:

   277         print 'Sorry - failed to write to local group cache '+treefile

   278         print 'Does it (or its directory) have the correct permissions?'

   279         sys.exit(1)

   280 

   281 # Return list of all groups on server -----------------------------

   282 

   283 def getallgroups(server):

   284     print 'Getting list of all groups...'

   285     treedate='010101'

   286     info = server.list()[1]

   287     groups = []

   288     print 'Processing...'

   289     if skipempty:

   290         print '\nIgnoring following empty groups:'

   291     for i in info:

   292         grpname = string.split(i[0])[0]

   293         if skipempty and string.atoi(i[1]) < string.atoi(i[2]):

   294             print grpname+' ',

   295         else:

   296             groups.append(grpname)

   297     print '\n'

   298     if skipempty:

   299         print '(End of empty groups)'

   300     return groups

   301 

   302 # Return list of new groups on server -----------------------------

   303 

   304 def getnewgroups(server, treedate):

   305     print 'Getting list of new groups since start of '+treedate+'...',

   306     info = server.newgroups(treedate,'000001')[1]

   307     print 'got %d.' % len(info)

   308     print 'Processing...',

   309     groups = []

   310     for i in info:

   311         grpname = string.split(i)[0]

   312         groups.append(grpname)

   313     print 'Done'

   314     return groups

   315 

   316 # Now the main program --------------------------------------------

   317 

   318 def main():

   319     global desc

   320 

   321     tree={}

   322 

   323     # Check that the output directory exists

   324     checkopdir(pagedir)

   325 

   326     try:

   327         print 'Connecting to '+newshost+'...'

   328         if sys.version[0] == '0':

   329             s = NNTP.init(newshost)

   330         else:

   331             s = NNTP(newshost)

   332         connected = 1

   333     except (nntplib.error_temp, nntplib.error_perm), x:

   334         print 'Error connecting to host:', x

   335         print 'I\'ll try to use just the local list.'

   336         connected = 0

   337 

   338     # If -a is specified, read the full list of groups from server

   339     if connected and len(sys.argv) > 1 and sys.argv[1] == '-a':

   340 

   341         groups = getallgroups(s)

   342 

   343     # Otherwise just read the local file and then add

   344     # groups created since local file last modified.

   345     else:

   346 

   347         (tree, treedate) = readlocallist(treefile)

   348         if connected:

   349             groups = getnewgroups(s, treedate)

   350 

   351     if connected:

   352         addtotree(tree, groups)

   353         writelocallist(treefile,tree)

   354 

   355     # Read group descriptions

   356     readdesc(descfile)

   357 

   358     print 'Creating pages...'

   359     createpage(rootpage, tree, '')

   360     print 'Done'

   361 

   362 if __name__ == "__main__":

   363     main()

   364 

   365 # That's all folks

   366 ######################################################################