Path: csiph.com!v102.xanadu-bbs.net!xanadu-bbs.net!feeder.erje.net!eu.feeder.erje.net!feeds.phibee-telecom.net!newsfeed.xs4all.nl!newsfeed1.news.xs4all.nl!xs4all!newsgate.cistron.nl!newsgate.news.xs4all.nl!post.news.xs4all.nl!not-for-mail Return-Path: X-Original-To: python-list@python.org Delivered-To: python-list@mail.python.org X-Spam-Status: OK 0.004 X-Spam-Evidence: '*H*': 0.99; '*S*': 0.00; 'encoding': 0.05; 'output': 0.05; '#if': 0.07; 'column': 0.07; 'ascii': 0.09; "subject:')": 0.09; 'subject:using': 0.09; 'try:': 0.09; 'django': 0.11; "'r',": 0.16; "'rb')": 0.16; 'ascii,': 0.16; 'codec': 0.16; 'csv': 0.16; 'err:': 0.16; 'subject:ignore': 0.16; 'wrote:': 0.18; 'do.': 0.18; 'seems': 0.21; '8bit%:5': 0.22; 'import': 0.22; 'to:name:python- list@python.org': 0.22; 'byte': 0.24; 'errors.': 0.24; 'skip:l 30': 0.24; 'subject: .': 0.24; 'skip:" 20': 0.27; 'header:In- Reply-To:1': 0.27; 'correct': 0.29; 'am,': 0.29; 'message- id:@mail.gmail.com': 0.30; 'code': 0.31; "skip:' 10": 0.31; '"': 0.31; "d'aprano": 0.31; 'prints': 0.31; 'steven': 0.31; 'skip:# 10': 0.33; 'skip:& 30': 0.33; 'subject:with': 0.35; "can't": 0.35; 'received:209.85': 0.35; 'except': 0.35; 'received:google.com': 0.35; '14,': 0.36; '8bit%:9': 0.36; 'received:209': 0.37; 'skip:& 10': 0.38; 'to:addr:python-list': 0.38; 'does': 0.39; 'to:addr:python.org': 0.39; '8bit%:6': 0.40; 'remove': 0.60; 'skip:u 10': 0.60; 'company': 0.60; 'skip:o 30': 0.61; 'save': 0.62; 'skip:\xe2 10': 0.65; 'here': 0.66; 'invalid': 0.68; '8bit%:21': 0.69; '8bit%:43': 0.74; '2015': 0.84; 'norm': 0.84 X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20130820; h=x-gm-message-state:mime-version:in-reply-to:references:from:date :message-id:subject:to:content-type; bh=3d4VPZxQD0ggBnauruKK2Gjj7/8YRZ74E2wnbLr0GxA=; b=H3mIN8jVgIxseSlmMi/pVqiv1V1dGpQPAU3QeoeZnA80edRpOU4HGqEz+nNQdk2pm2 ywXR1LfRctzEfDI7mDo+AhIcQl3gkk24N6HuFMEwRWCDdQPy6rZc2Ny0Jm4hOjQuoI+7 XEDWG2H554+QWwc3/m82eHyUQAZG9R4ieS+GhDQXW6k4heBH5ECqPSbYNVFHqBB3Q+n7 icOJs/3hCeuUUAzYwu0B9vjmEc7JQTT5vzxzFau44y2NVzo1dSuKg1r9tbmvtsqC1p4B w//9l8BrTQgCMAlXtEAUPju17WrX+i9UzRIkR57yTVgLOxEV+pwNmqDlrz7cydIlL8V9 JwmQ== X-Gm-Message-State: ALoCoQmJhjuG7XXwEzrb+VVSX6BE21MIX8Eh6cYIYvEFx+7fO3jGFrZ5mCRlevmY+XT1BKSBvBmM X-Received: by 10.60.150.147 with SMTP id ui19mr1423867oeb.82.1429023134168; Tue, 14 Apr 2015 07:52:14 -0700 (PDT) MIME-Version: 1.0 In-Reply-To: <552d1aab$0$12996$c3e8da3$5496439d@news.astraweb.com> References: <552d14c0$0$12991$c3e8da3$5496439d@news.astraweb.com> <552d1aab$0$12996$c3e8da3$5496439d@news.astraweb.com> From: Vincent Davis Date: Tue, 14 Apr 2015 08:51:53 -0600 Subject: Re: using DictReader() with .decode('utf-8', 'ignore') To: "python-list@python.org" Content-Type: multipart/alternative; boundary=047d7b5d41ce49dbe80513b06008 X-BeenThere: python-list@python.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: General discussion list for the Python programming language List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Newsgroups: comp.lang.python Message-ID: Lines: 193 NNTP-Posting-Host: 2001:888:2000:d::a6 X-Trace: 1429023145 news.xs4all.nl 2931 [2001:888:2000:d::a6]:53582 X-Complaints-To: abuse@xs4all.nl Xref: csiph.com comp.lang.python:88966 --047d7b5d41ce49dbe80513b06008 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: quoted-printable On Tue, Apr 14, 2015 at 7:48 AM, Steven D'Aprano < steve+comp.lang.python@pearwood.info> wrote: > with open(dfile, 'rb') as f: > for line in f: > try: > s =3D line.decode('utf-8', 'strict') > except UnicodeDecodeError as err: > print(err) > > If you need help deciphering the errors, please copy and paste them here > and > we'll see what we can do. Below are the errors. I knew about these and I think the correct encoding is windows-1252. I will paste some code and output at the end of this email that prints the offending column in the line. These are very likely errors, and so I what to remove them. I am reading this csv into django sqlite3 db. What is strange to me is that using =E2=80=8B"=E2=80=8B with open(dfile, 'r', encoding=3D'utf-8', errors=3D'ignore', newline=3D'') =E2=80=8B"=E2=80=8B does not seem to remove these =E2=80=8B, it seems to correctly save them to the db which I don't understa= nd.=E2=80=8B =E2=80=8B 'utf-8' codec can't decode byte 0xa6 in position 368: invalid start byte 'utf-8' codec can't decode byte 0xac in position 223: invalid start byte 'utf-8' codec can't decode byte 0xa6 in position 1203: invalid start byte 'utf-8' codec can't decode byte 0xa2 in position 44: invalid start byte 'utf-8' codec can't decode byte 0xac in position 396: invalid start byte import chardet with open("DATA/ATSDTA_ATSP600.csv", 'rb') as f: for line in f: code =3D chardet.detect(line) #if code =3D=3D {'confidence': 0.5, 'encoding': 'windows-1252'}: if code !=3D {'encoding': 'ascii', 'confidence': 1.0}: print(code) win =3D line.decode('windows-1252').split(',') #windows-1252 norm =3D line.decode('utf-8', 'ignore').split(',') ascii =3D line.decode('ascii', "ignore").split(',') ascii2 =3D line.decode('ISO-8859-1').split(',') for w, n, a, a2 in zip(win, norm, ascii, ascii2): if w !=3D n: print(w =E2=80=8B) =E2=80=8B print( n =E2=80=8B) =E2=80=8B a, a2) print(win[0]) =E2=80=8B## Output=E2=80=8B {'encoding': 'windows-1252', 'confidence': 0.5} "=C2=A6 " " " " " "=C2=A6 " "040543" {'encoding': 'windows-1252', 'confidence': 0.5} "LEASE GREGPRU D =C2=ACETERSPM " "LEASE GREGPRU D ETERSPM " "LEASE GREGPRU D ETERSPM " "LEASE GREGPRU D =C2=ACETERSPM " "979643" {'encoding': 'windows-1252', 'confidence': 0.5} "=C2=A6 " " " " " "=C2=A6 " "986979" {'encoding': 'windows-1252', 'confidence': 0.5} "WELLS FARGO &=C2=A2 COMPANY " "WELLS FARGO & COMPANY " "WELLS FARGO & COMPANY " "WELLS FARGO &=C2=A2 COMPANY " "994946" {'encoding': 'windows-1252', 'confidence': 0.5} OSSOSSO=C2=AC=C2=ACO " OSSOSSOO " OSSOSSOO " OSSOSS= O=C2=AC=C2=ACO " "996535" Vincent Davis 720-301-3003 --047d7b5d41ce49dbe80513b06008 Content-Type: text/html; charset=UTF-8 Content-Transfer-Encoding: quoted-printable

= On Tue, Apr 14, 2015 at 7:48 AM, Steven D'Aprano <= s= teve+comp.lang.python@pearwood.info> wrote:
with open(dfile, 'rb') as f:
=C2=A0 =C2=A0 for line in f:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 try:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 s =3D line.decode('utf-8'= , 'strict')
=C2=A0 =C2=A0 =C2=A0 =C2=A0 except UnicodeDecodeError as err:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 print(err)

If you need help deciphering the errors, please copy and paste them here an= d
we'll see what we can do.

Below are the errors. I= knew about these and I think the correct encoding is windows-1252. I will = paste some code and output at the end of this email that prints the offendi= ng column in the line. These are very likely errors, and so I what to remov= e them. I am reading this csv into django sqlite3 db. What is strange to me= is that using
=E2=80=8B"=E2=80=8B
with= open(dfile, 'r', encoding=3D'utf-8', errors=3D'ignore&= #39;, newline=3D'')
=E2=80=8B"=E2=80= =8B
=C2=A0does not seem to remove these
=E2= =80=8B, it seems to correctly save them to the db which I don't underst= and.=E2=80=8B
=E2=80=8B


'ut= f-8' codec can't decode byte 0xa6 in position 368: invalid start by= te
'utf-8' codec can't decode byte 0xac in position 223: inv= alid start byte
'utf-8' codec can't decode byte 0xa6 in posi= tion 1203: invalid start byte
'utf-8' codec can't decode byt= e 0xa2 in position 44: invalid start byte
'utf-8' codec can'= t decode byte 0xac in position 396: invalid start byte

import chardet
with open("DATA/ATSDTA= _ATSP600.csv", 'rb') as f:
=C2= =A0 =C2=A0 for line in f:
=C2=A0 =C2=A0 =C2= =A0 =C2=A0 code =3D chardet.detect(line)
= =C2=A0 =C2=A0 =C2=A0 =C2=A0 #if code =3D=3D {'confidence': 0.5, = 9;encoding': 'windows-1252'}:
= =C2=A0 =C2=A0 =C2=A0 =C2=A0 if code !=3D {'encoding': 'ascii= 9;, 'confidence': 1.0}:
=C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 print(code)
=C2=A0 =C2=A0 =C2=A0 =C2=A0 win =3D line.decode('windows-1252').sp= lit(',') #windows-1252
=C2=A0 =C2= =A0 =C2=A0 =C2=A0 norm =3D line.decode('utf-8', 'ignore').s= plit(',')
=C2=A0 =C2=A0 =C2=A0 =C2= =A0 ascii =3D line.decode('ascii', "ignore").split(',= ')
=C2=A0 =C2=A0 =C2=A0 =C2=A0 ascii2 = =3D line.decode('ISO-8859-1').split(',')
=C2=A0 =C2=A0 =C2=A0 =C2=A0=C2=A0
=C2=A0 =C2=A0 =C2=A0 =C2=A0 for w, n, a, a2 in zip(win, norm, ascii, a= scii2):
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 if w !=3D n:
=C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 print(w
=E2=80= =8B)
=E2=80= =8B =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 print(
n
=E2=80=8B)
=E2=80=8B
a, a2)
=C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 print(win[0])

=E2=80=8B#= # Output=E2=80=8B

{'encoding': 'windows-1252', 'confidence&#=
39;: 0.5}
"=C2=A6   " "   " "   " "=C2=A6   "
"040543"
{'encoding': 'windows-1252', 'confidence': 0.5}
"LEASE GREGPRU D =C2=ACETERSPM                 " "LEASE GREG=
PRU D ETERSPM                 " "LEASE GREGPRU D ETERSPM         =
        " "LEASE GREGPRU D =C2=ACETERSPM                 "
"979643"
{'encoding': 'windows-1252', 'confidence': 0.5}
"=C2=A6   " "   " "   " "=C2=A6   "
"986979"
{'encoding': 'windows-1252', 'confidence': 0.5}
"WELLS FARGO &=C2=A2 COMPANY                   " "WELLS =
FARGO & COMPANY                   " "WELLS FARGO & COMPAN=
Y                   " "WELLS FARGO &=C2=A2 COMPANY           =
        "
"994946"
{'encoding': 'windows-1252', 'confidence': 0.5}
OSSOSSO=C2=AC=C2=ACO         " OSSOSSOO         " OSSOSSOO       =
  " OSSOSSO=C2=AC=C2=ACO         "
"996535"


Vincent Davis
720-301-3003
--047d7b5d41ce49dbe80513b06008--