Path: csiph.com!v102.xanadu-bbs.net!xanadu-bbs.net!goblin1!goblin2!goblin.stu.neva.ru!newsfeed.xs4all.nl!newsfeed3a.news.xs4all.nl!xs4all!post.news.xs4all.nl!not-for-mail
MIME-Version: 1.0
Date: Sat, 25 Jan 2014 11:00:41 -0800
Subject: Pls help me...I want to save data to my database but I am unable to
From: Max Cuban <edzeame@gmail.com>
To: python-list@python.org
Content-Type: multipart/alternative; boundary=bcaec520e87d4535bb04f0d017bc
Precedence: list
Newsgroups: comp.lang.python
Message-ID: <mailman.5978.1390676453.18130.python-list@python.org>
Lines: 195
NNTP-Posting-Host: 2001:888:2000:d::a6
Xref: csiph.com comp.lang.python:64749

--bcaec520e87d4535bb04f0d017bc
Content-Type: text/plain; charset=ISO-8859-1

This is my first programming pet project. I have the following script that
extracts links from specific sites and display them on the web(via django).
The script work fine but I'm unable to save any stuff in my database.

Hence if I run the code, I get the output I want but then it always
extracts only new content. I will rather want to have the content scrapped
earlier saved to the database so that on subsequent run, it only scrap and
append ONLY new links to the list.
[ ]
Any help will be appreciated.
[]
    # Create your views here.
    from django.template.loader import get_template
    from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
    from django.shortcuts import render_to_response
    from django.template import Context
    from bs4 import BeautifulSoup
    import urllib2, sys
    import urlparse
    import re
    from datetime import date, datetime
    from listing.models import jobLinks


    def businessghana():
        site = "http://www.businessghana.com/portal/jobs"
        hdr = {'User-Agent' : 'Mozilla/5.0'}
        req = urllib2.Request(site, headers=hdr)
        jobpass = urllib2.urlopen(req)
        soup = BeautifulSoup(jobpass)
        for tag in soup.find_all('a', href = True):
            tag['href'] = urlparse.urljoin('
http://www.businessghana.com/portal/', tag['href'])
        return map(str, soup.find_all('a', href =
re.compile('.getJobInfo')))

    def tonaton():
        site = "http://tonaton.com/en/job-vacancies-in-ghana"
        hdr = {'User-Agent' : 'Mozilla/5.0'}
        req = urllib2.Request(site, headers=hdr)
        jobpass = urllib2.urlopen(req)
        soup = BeautifulSoup(jobpass)
        result = []
        # next two lines make all the links in the soup absolute
        for tag in soup.find_all('a', href=True):
            tag['href'] = urlparse.urljoin('http://www.tonaton.com',
tag['href'])
        # assign all 'h2' tags to 'jobs'. The 'h2'tag contains the required
links
        jobs = soup.find_all('h2')
        # Loop through the 'h2' tags and extract all the links
        for h2 in soup.find_all('h2'):
            n = h2.next_element
            if n.name == 'a':  result.append(str(n))
        return result

    def jobscomgh():
        site = "http://jobs.com.gh"
        hdr = {'User-Agent' : 'Mozilla/5.0'}
        req = urllib2.Request(site, headers=hdr)
        jobpass = urllib2.urlopen(req)
        soup = BeautifulSoup(jobpass)
        return map(str, soup.find_all('a', href =
re.compile('.display-job')))

    businessghana_links = businessghana()
    tonaton_links = tonaton()
    jobscomgh_links = jobscomgh()

    def all_links():
        return (businessghana_links + tonaton_links + jobscomgh_links)

    def save_new_links(all_links):
        current_links = jobLinks.objects.all()
        for i in all_links:
            if i not in current_links:
                jobLinks.objects.create(url=i)

    def this_week_links(all_links):
        return jobLinks.objects.filter(date__gte =
datetime.timedelta(days=-7))

    save_new_links(all_links)
    this_week_links(all_links)

    def display_links(request):
        name = all_links()
        paginator = Paginator(name, 25)
        page = request.GET.get('page')
        try:
            name = paginator.page(page)
        except PageNotAnInteger:
            name = paginator.page(1)
        except EmptyPage:
            name = paginator.page(paginator.num_pages)


        return render_to_response('jobs.html', {'name' : name})

--bcaec520e87d4535bb04f0d017bc
Content-Type: text/html; charset=ISO-8859-1
Content-Transfer-Encoding: quoted-printable

<div dir=3D"ltr"><div>This is my first programming pet project. I have the =
following script that extracts links from specific sites and display them o=
n the web(via django). The script work fine but I&#39;m unable to save any =
stuff in my database.</div>
<div><br></div><div>Hence if I run the code, I get the output I want but th=
en it always extracts only new content. I will rather want to have the cont=
ent scrapped earlier saved to the database so that on subsequent run, it on=
ly scrap and append ONLY new links to the list.</div>
<div>[ ]</div><div>Any help will be appreciated.</div><div>[]</div><div>=A0=
 =A0 # Create your views here.</div><div>=A0 =A0 from django.template.loade=
r import get_template</div><div>=A0 =A0 from django.core.paginator import P=
aginator, EmptyPage, PageNotAnInteger</div>
<div>=A0 =A0 from django.shortcuts import render_to_response=A0</div><div>=
=A0 =A0 from django.template import Context</div><div>=A0 =A0 from bs4 impo=
rt BeautifulSoup</div><div>=A0 =A0 import urllib2, sys</div><div>=A0 =A0 im=
port urlparse</div>
<div>=A0 =A0 import re</div><div>=A0 =A0 from datetime import date, datetim=
e</div><div>=A0 =A0 from listing.models import jobLinks</div><div><br></div=
><div><br></div><div>=A0 =A0 def businessghana():</div><div>=A0 =A0 =A0 =A0=
 site =3D &quot;<a href=3D"http://www.businessghana.com/portal/jobs">http:/=
/www.businessghana.com/portal/jobs</a>&quot;</div>
<div>=A0 =A0 =A0 =A0 hdr =3D {&#39;User-Agent&#39; : &#39;Mozilla/5.0&#39;}=
</div><div>=A0 =A0 =A0 =A0 req =3D urllib2.Request(site, headers=3Dhdr)</di=
v><div>=A0 =A0 =A0 =A0 jobpass =3D urllib2.urlopen(req)</div><div>=A0 =A0 =
=A0 =A0 soup =3D BeautifulSoup(jobpass)</div>
<div>=A0 =A0 =A0 =A0 for tag in soup.find_all(&#39;a&#39;, href =3D True):<=
/div><div>=A0 =A0 =A0 =A0 =A0 =A0 tag[&#39;href&#39;] =3D urlparse.urljoin(=
&#39;<a href=3D"http://www.businessghana.com/portal/">http://www.businessgh=
ana.com/portal/</a>&#39;, tag[&#39;href&#39;])</div>
<div>=A0 =A0 =A0 =A0 return map(str, soup.find_all(&#39;a&#39;, href =3D re=
.compile(&#39;.getJobInfo&#39;)))</div><div><br></div><div>=A0 =A0 def tona=
ton():</div><div>=A0 =A0 =A0 =A0 site =3D &quot;<a href=3D"http://tonaton.c=
om/en/job-vacancies-in-ghana">http://tonaton.com/en/job-vacancies-in-ghana<=
/a>&quot;</div>
<div>=A0 =A0 =A0 =A0 hdr =3D {&#39;User-Agent&#39; : &#39;Mozilla/5.0&#39;}=
</div><div>=A0 =A0 =A0 =A0 req =3D urllib2.Request(site, headers=3Dhdr)</di=
v><div>=A0 =A0 =A0 =A0 jobpass =3D urllib2.urlopen(req)</div><div>=A0 =A0 =
=A0 =A0 soup =3D BeautifulSoup(jobpass)</div>
<div>=A0 =A0 =A0 =A0 result =3D []</div><div>=A0 =A0 =A0 =A0 # next two lin=
es make all the links in the soup absolute =A0 =A0</div><div>=A0 =A0 =A0 =
=A0 for tag in soup.find_all(&#39;a&#39;, href=3DTrue):</div><div>=A0 =A0 =
=A0 =A0 =A0 =A0 tag[&#39;href&#39;] =3D urlparse.urljoin(&#39;<a href=3D"ht=
tp://www.tonaton.com">http://www.tonaton.com</a>&#39;, tag[&#39;href&#39;])=
</div>
<div>=A0 =A0 =A0 =A0 # assign all &#39;h2&#39; tags to &#39;jobs&#39;. The =
&#39;h2&#39;tag contains the required links =A0=A0</div><div>=A0 =A0 =A0 =
=A0 jobs =3D soup.find_all(&#39;h2&#39;)</div><div>=A0 =A0 =A0 =A0 # Loop t=
hrough the &#39;h2&#39; tags and extract all the links</div>
<div>=A0 =A0 =A0 =A0 for h2 in soup.find_all(&#39;h2&#39;):</div><div>=A0 =
=A0 =A0 =A0 =A0 =A0 n =3D h2.next_element</div><div>=A0 =A0 =A0 =A0 =A0 =A0=
 if <a href=3D"http://n.name">n.name</a> =3D=3D &#39;a&#39;: =A0result.appe=
nd(str(n))</div><div>=A0 =A0 =A0 =A0 return result</div>
<div><br></div><div>=A0 =A0 def jobscomgh():</div><div>=A0 =A0 =A0 =A0 site=
 =3D &quot;<a href=3D"http://jobs.com.gh">http://jobs.com.gh</a>&quot;</div=
><div>=A0 =A0 =A0 =A0 hdr =3D {&#39;User-Agent&#39; : &#39;Mozilla/5.0&#39;=
}</div><div>=A0 =A0 =A0 =A0 req =3D urllib2.Request(site, headers=3Dhdr)</d=
iv>
<div>=A0 =A0 =A0 =A0 jobpass =3D urllib2.urlopen(req)</div><div>=A0 =A0 =A0=
 =A0 soup =3D BeautifulSoup(jobpass)</div><div>=A0 =A0 =A0 =A0 return map(s=
tr, soup.find_all(&#39;a&#39;, href =3D re.compile(&#39;.display-job&#39;))=
)</div><div><br></div><div>
=A0 =A0 businessghana_links =3D businessghana()</div><div>=A0 =A0 tonaton_l=
inks =3D tonaton()</div><div>=A0 =A0 jobscomgh_links =3D jobscomgh()</div><=
div><br></div><div>=A0 =A0 def all_links():</div><div>=A0 =A0 =A0 =A0 retur=
n (businessghana_links + tonaton_links + jobscomgh_links)</div>
<div><br></div><div>=A0 =A0 def save_new_links(all_links):</div><div>=A0 =
=A0 =A0 =A0 current_links =3D jobLinks.objects.all()</div><div>=A0 =A0 =A0 =
=A0 for i in all_links:</div><div>=A0 =A0 =A0 =A0 =A0 =A0 if i not in curre=
nt_links:</div><div>=A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 jobLinks.objects.create=
(url=3Di)</div>
<div>=A0 =A0=A0</div><div>=A0 =A0 def this_week_links(all_links):</div><div=
>=A0 =A0 =A0 =A0 return jobLinks.objects.filter(date__gte =3D datetime.time=
delta(days=3D-7))</div><div><br></div><div>=A0 =A0 save_new_links(all_links=
)</div><div>=A0 =A0 this_week_links(all_links) =A0 =A0 =A0 =A0 =A0 =A0 =A0 =
=A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0</div>
<div><br></div><div>=A0 =A0 def display_links(request):</div><div>=A0 =A0 =
=A0 =A0 name =3D all_links()</div><div>=A0 =A0 =A0 =A0 paginator =3D Pagina=
tor(name, 25)</div><div>=A0 =A0 =A0 =A0 page =3D request.GET.get(&#39;page&=
#39;)</div><div>=A0 =A0 =A0 =A0 try:</div>
<div>=A0 =A0 =A0 =A0 =A0 =A0 name =3D paginator.page(page)</div><div>=A0 =
=A0 =A0 =A0 except PageNotAnInteger:</div><div>=A0 =A0 =A0 =A0 =A0 =A0 name=
 =3D paginator.page(1)</div><div>=A0 =A0 =A0 =A0 except EmptyPage:</div><di=
v>=A0 =A0 =A0 =A0 =A0 =A0 name =3D paginator.page(paginator.num_pages)</div=
>
<div>=A0 =A0=A0</div><div>=A0 =A0 =A0 =A0=A0</div><div>=A0 =A0 =A0 =A0 retu=
rn render_to_response(&#39;jobs.html&#39;, {&#39;name&#39; : name}) =A0 =A0=
</div><div>=A0 =A0=A0</div><div><br></div></div>

--bcaec520e87d4535bb04f0d017bc--