Path: csiph.com!usenet.pasdenom.info!aioe.org!news.stack.nl!newsfeed.xs4all.nl!newsfeed5.news.xs4all.nl!xs4all!post.news.xs4all.nl!not-for-mail
Date: Tue, 24 Jul 2012 13:10:45 +0100
From: MRAB <python@mrabarnett.plus.com>
User-Agent: Mozilla/5.0 (Windows NT 5.1; rv:14.0) Gecko/20120713 Thunderbird/14.0
MIME-Version: 1.0
To: python-list@python.org
Subject: Re: groveling over a file for Q:: and A:: stmts
References: <4851867b-6c8e-41e4-8d39-56e10e193a12@googlegroups.com>
In-Reply-To: <4851867b-6c8e-41e4-8d39-56e10e193a12@googlegroups.com>
Content-Type: text/plain; charset=ISO-8859-1; format=flowed
Content-Transfer-Encoding: 7bit
Precedence: list
Reply-To: python-list@python.org
Newsgroups: comp.lang.python
Message-ID: <mailman.2530.1343131847.4697.python-list@python.org>
Lines: 94
NNTP-Posting-Host: 2001:888:2000:d::a6
Xref: csiph.com comp.lang.python:25980

On 24/07/2012 08:50, paul618 wrote:
> #!/usr/bin/env python
> # grep_for_QA.py  I am only looking to isolate uniq Q:: and A:: stmts from my daily files
> #
> # note:  This algorithm will fail if there are any blank lines within the Q and A area of interest (a paragraph)
>
> # D. Beazley is my fav documentation
>
> import re, glob
> import pprint as pp
>
> sampledata = '''
> A:: And Straight Street is playin on the Radio Free Tibet.  What are the chances, DTMB?
> Q:: About 1 in 518400, Professor.
> A:: Correct!  Err, I thought it was 1:410400, but <i>close enough for jazz!</i>
>
>
> '''
>
> pattern0 = re.compile("Q::")
> pattern1 = re.compile("A::")  # objects of interest can start with A:: ;; not alway Q::
> END_OF_PARAGRAPH_pat = "\n\s*\n"
>
> path = "/Users/paultaney/dailies2012/0722" # an example of real data set.
>
> toggle = False
> L = []
> M = []
>
> #file = open(path, "r")
> try:
>      #for line in file.readlines():
>      for line in sampledata:

sampledata is a string, therefore this is iterating over the string,
which yields characters, not lines. Try using sampledata.splitlines():

     for line in sampledata.splitlines():

>          try:
>              # Later, I also need to treat Unicode -- and I am clueless.
>
>              # falsestarts::
>              #line.encode("utf8").decode('xxx', 'ignore')
>              #line.encode("utf8", 'ignore')
>              #line.decode('8859')
>              #line.decode('8859')  # 8859, Latin-1 doesn't cover my  CJK pastings  AT ALL
>              #line.decode('GB18030')  # 171006 -- ack
>              #encoded_line = line  # xxx line.encode("utf8")
>
>              mo0 = re.search(pattern0, line)

This searches for pattern0 anywhere in the line. You really want to
check whether the line starts with pattern0, which is better done with:

     line.startswith("Q::")

>              mo1 = re.search(pattern1, line)
>              mo2 = re.search(END_OF_PARAGRAPH_pat, line)
>
>              if mo0:
>                  if 1: print ("I see pattern 0")
>                  toggle = True
>                  if 1: print(line)
>                  M.append(mo0.group())
>
>              if mo1:
>                  if 1: print ("I see pattern 1")
>                  toggle = True
>                  M.append(mo1.group())
>
>              if mo2 and toggle:
>                  if 1: print ("I see pattern 2 AND toggle is set")
>                  # got one.  save it for uniqifying, and empty the container
>                  toggle = False
>                  L.append(M)
>                  M = []
>
>          except Exception as e:
>              print("--- " + e + " ---")
>
> except UnicodeDecodeError:
>      #encoded_line = encoded_line.urlsafe_b64encode(re.replace("asdf", encoded_line))
>      #line = re.sub(".+", "--- asdf ---", line)
>      pass
>
> L.sort
> print (L)
>
> # and what"s wrong with some of this, here!
> #myHash = set(L)    # uniqify
> #pp.pprint(myHash)  # july 23, 131001 hike!
>