Commit b724e6010e4acbbf5e4d212acfc67db2be861cff

  • avatar
  • arvind
  • Sat Apr 05 18:13:13 IST 2014
Fix malformed how of re-narration type. `how` has blogUrl key now
which contains the permalink of the post.
Fix PEP8 warnings and errors and on the alipi.py code.
alipi/alipi.py
(147 / 77)
  
88import urllib2
99import StringIO
1010from flask import g
11from flask import redirect
1211from urllib import quote_plus
1312from urllib import unquote_plus
1413import conf
1616import requests
1717from flask import jsonify
1818import json
19from flask import url_for
2019
20
2121app = Flask(__name__)
22
23
2224@app.before_request
2325def first():
24 g.connection = pymongo.MongoClient('localhost',27017) #Create the object once and use it.
26 g.connection = pymongo.MongoClient('localhost', 27017) # Create the
27 # object once and use it.
2528 g.db = g.connection[conf.MONGODB[0]]
2629
27# @app.after_request
28# def set_secret(response):
29# response.set_cookie("key", conf.SWEET_SECRET_KEY[0])
3030
31
3231@app.teardown_request
3332def close(exception):
3433 g.connection.disconnect()
3534
3635
3736@app.route('/')
38def start_page() :
37def start_page():
3938 d = {}
4039 d['foruri'] = request.args['foruri']
41 myhandler1 = urllib2.Request(d['foruri'],headers={'User-Agent':"Mozilla/5.0 (X11; Linux x86_64; rv:25.0) Gecko/20100101 Firefox/25.0)"}) #A fix to send user-agents, so that sites render properly.
40 myhandler1 = urllib2.Request(d['foruri'],
41 headers={'User-Agent':
42 "Mozilla/5.0 (X11; " +
43 "Linux x86_64; rv:25.0)" +
44 "Gecko/20100101 Firefox/25.0)"})
45 # A fix to send user-agents, so that sites render properly.
4246 try:
4347 a = urllib2.urlopen(myhandler1)
4448 if a.geturl() != d['foruri']:
45 return "There was a server redirect, please click on the <a href='http://y.a11y.in/web?foruri={0}'>link</a> to continue.".format(quote_plus(a.geturl()))
49 return ("There was a server redirect, please click on the" +
50 " <a href='http://y.a11y.in/web?foruri={0}'>link</a> to" +
51 " continue.".format(quote_plus(a.geturl())))
4652 else:
4753 page = a.read()
4854 a.close()
4955 except ValueError:
50 return "The link is malformed, click <a href='http://y.a11y.in/web?foruri={0}&lang={1}&interactive=1'>here</a> to be redirected.".format(quote_plus(unquote_plus(d['foruri'].encode('utf-8'))),request.args['lang'])
56 return ("The link is malformed, click " +
57 "<a href='http://y.a11y.in/web?foruri={0}&lang={1}" +
58 "&interactive=1'>" +
59 "here</a> to be redirected.".format(
60 quote_plus(unquote_plus(d['foruri'].encode('utf-8'))),
61 request.args['lang']))
5162 except urllib2.URLError:
5263 return render_template('error.html')
5364 try:
54 page = unicode(page,'utf-8') #Hack to fix improperly displayed chars on wikipedia.
65 page = unicode(page, 'utf-8') # Hack to fix improperly displayed chars on wikipedia.
5566 except UnicodeDecodeError:
56 pass #Some pages may not need be utf-8'ed
67 pass # Some pages may not need be utf-8'ed
5768 try:
5869 g.root = lxml.html.parse(StringIO.StringIO(page)).getroot()
5970 except ValueError:
60 g.root = lxml.html.parse(d['foruri']).getroot() #Sometimes creators of the page lie about the encoding, thus leading to this execption. http://lxml.de/parsing.html#python-unicode-strings
71 g.root = lxml.html.parse(d['foruri']).getroot() # Sometimes creators of the page lie about the encoding, thus leading to this execption. http://lxml.de/parsing.html#python-unicode-strings
6172 if request.args.has_key('lang') == False and request.args.has_key('blog') == False:
62 g.root.make_links_absolute(d['foruri'], resolve_base_href = True)
73 g.root.make_links_absolute(d['foruri'],
74 resolve_base_href=True)
6375 for i in g.root.iterlinks():
6476 if i[1] == 'href' and i[0].tag != 'link':
6577 try:
66 i[0].attrib['href'] = 'http://{0}?foruri={1}'.format(conf.DEPLOYURL[0],quote_plus(i[0].attrib['href']))
78 i[0].attrib['href'] = 'http://{0}?foruri={1}'.format(
79 conf.DEPLOYURL[0], quote_plus(i[0].attrib['href']))
6780 except KeyError:
68 i[0].attrib['href'] = '{0}?foruri={1}'.format(conf.DEPLOYURL[0],quote_plus(i[0].attrib['href'].encode('utf-8')))
81 i[0].attrib['href'] = '{0}?foruri={1}'.format(
82 conf.DEPLOYURL[0], quote_plus(
83 i[0].attrib['href'].encode('utf-8')))
6984 setScripts()
70 g.root.body.set("onload","a11ypi.loadOverlay();")
85 g.root.body.set("onload", "a11ypi.loadOverlay();")
7186 response = make_response()
7287 response.data = lxml.html.tostring(g.root)
7388 return response
9090 elif request.args.has_key('lang') == True and request.args.has_key('interactive') == True and request.args.has_key('blog') == False:
9191 setScripts()
9292 setSocialScript()
93 g.root.body.set("onload","a11ypi.ren();a11ypi.tweet(); a11ypi.facebook(); a11ypi.loadOverlay();")
94 g.root.make_links_absolute(d['foruri'], resolve_base_href = True)
93 g.root.body.set("onload", "a11ypi.ren();a11ypi.tweet(); " +
94 "a11ypi.facebook(); a11ypi.loadOverlay();")
95 g.root.make_links_absolute(d['foruri'], resolve_base_href=True)
9596 response = make_response()
9697 response.data = lxml.html.tostring(g.root)
9798 return response
9899
99
100100 elif request.args.has_key('lang') == True and request.args.has_key('blog') == False:
101101 script_jq_mini = g.root.makeelement('script')
102102 g.root.body.append(script_jq_mini)
107107 g.root.body.append(script_test)
108108 script_test.set("src", conf.APPURL[0] + "/alipi/ui.js")
109109 script_test.set("type", "text/javascript")
110 g.root.body.set("onload","a11ypi.ren()");
110 g.root.body.set("onload", "a11ypi.ren()")
111111 response = make_response()
112112 response.data = lxml.html.tostring(g.root)
113113 return response
114114
115
116115 elif request.args.has_key('interactive') == True and request.args.has_key('blog') == True and request.args.has_key('lang') == True:
117116 setScripts()
118117 setSocialScript()
119 g.root.body.set("onload","a11ypi.filter(); a11ypi.tweet(); a11ypi.facebook(); a11ypi.loadOverlay();");
120 g.root.make_links_absolute(d['foruri'], resolve_base_href = True)
118 g.root.body.set("onload", "a11ypi.filter(); a11ypi.tweet();" +
119 "a11ypi.facebook(); a11ypi.loadOverlay();")
120 g.root.make_links_absolute(d['foruri'], resolve_base_href=True)
121121 response = make_response()
122122 response.data = lxml.html.tostring(g.root)
123123 return response
124124
125125 elif request.args.has_key('interactive') == False and request.args.has_key('blog') == True:
126126 setScripts()
127 g.root.make_links_absolute(d['foruri'], resolve_base_href = True)
127 g.root.make_links_absolute(d['foruri'], resolve_base_href=True)
128128 g.root.body.set('onload', 'a11ypi.loadOverlay();')
129129 response = make_response()
130130 response.data = lxml.html.tostring(g.root)
143143
144144 style = g.root.makeelement('link')
145145 g.root.body.append(style)
146 style.set("rel","stylesheet")
146 style.set("rel", "stylesheet")
147147 style.set("type", "text/css")
148148 style.set("href", conf.APPURL[0] + "/alipi/pack.min.css")
149149
154154 info_button.set("id", "info")
155155 info_button.set("class", "alipi")
156156 info_button.set("onClick", "a11ypi.showInfo(a11ypi.responseJSON);")
157 info_button.text = "Info"
158 info_button.set("title", "Have a look at the information of each renarrated element")
157 info_button.text = "Info"
158 info_button.set("title", "Have a look at the information of each" +
159 " renarrated element")
159160
160161 share_button = g.root.makeelement('button')
161162 g.root.body.append(share_button)
162163 share_button.set("id", "share")
163164 share_button.set("class", "alipi")
164165 share_button.set("onClick", "a11ypi.share();")
165 share_button.text = "Share"
166 share_button.text = "Share"
166167 share_button.set("title", "Share your contribution in your social network")
167168
168169 see_orig = g.root.makeelement('button')
172172 see_orig.set("class", "alipi")
173173 see_orig.set("onClick", "a11ypi.showOriginal();")
174174 see_orig.text = "Original Page"
175 see_orig.set("title", "Go to Original link, the original page of this renarrated")
175 see_orig.set("title", "Go to Original link, the original page of this" +
176 " renarrated page")
176177
177178 tweetroot = g.root.makeelement("div")
178179 tweetroot.set("id", "tweet-root")
187187 tweet.set("class", "alipi twitter-share-button")
188188 tweet.set("data-via", "a11ypi")
189189 tweet.set("data-lang", "en")
190 tweet.set("data-url", "http://y.a11y.in/web?foruri={0}&lang={1}&interactive=1".format(quote_plus(request.args['foruri']),(request.args['lang']).encode('unicode-escape')))
190 tweet.set("data-url", "http://y.a11y.in/web?foruri={0}&lang={1}&" +
191 "interactive=1".format(quote_plus(request.args['foruri']),
192 (request.args['lang']).encode(
193 'unicode-escape')))
191194 tweet.textContent = "Tweet"
192195 tweetroot.append(tweet)
193196
198198 fblike.set("id", "fb-like")
199199 fblike.set("class", "alipi fb-like")
200200 fblike.set("style", "display:none;padding:10px;")
201 fblike.set("data-href", "http://y.a11y.in/web?foruri={0}&lang={1}&interactive=1".format(quote_plus(request.args['foruri']),(request.args['lang']).encode('unicode-escape')))
201 fblike.set("data-href", "http://y.a11y.in/web?foruri={0}&lang={1}&" +
202 "interactive=1".format(quote_plus(request.args['foruri']),
203 (request.args['lang']).encode(
204 'unicode-escape')))
202205 fblike.set("data-send", "true")
203206 fblike.set("data-layout", "button_count")
204207 fblike.set("data-width", "50")
211211
212212 style = g.root.makeelement('link')
213213 g.root.body.append(style)
214 style.set("rel","stylesheet")
214 style.set("rel", "stylesheet")
215215 style.set("type", "text/css")
216216 style.set("href", "http://y.a11y.in/alipi/stylesheet.css")
217217
220220def show_directory():
221221 collection = g.db['post']
222222 query = collection.group(
223 key = Code('function(doc){return {"about" : doc.about,"lang":doc.lang}}'),
224 condition={"about":{'$regex':'^[/\S/]'}},
223 key=Code('function(doc){' +
224 'return {"about" : doc.about,"lang":doc.lang}}'),
225 condition={"about": {'$regex': '^[/\S/]'}},
225226 initial={'na': []},
226227 reduce=Code('function(doc,out){out.na.push(doc.blog)}')
227228 )
228229 query.reverse()
229 return render_template('directory.html', name=query, mymodule = quote_plus, myset=set, mylist= list)
230 return render_template('directory.html', name=query, mymodule=quote_plus,
231 myset=set, mylist=list)
230232
233
231234@app.route('/getLoc', methods=['GET'])
232235def get_loc():
233236
234237 term = request.args['term']
235 connection = oursql.Connection(conf.DBHOST[0],conf.DBUSRNAME[0],conf.DBPASSWD[0],db=conf.DBNAME[0])
238 connection = oursql.Connection(conf.DBHOST[0], conf.DBUSRNAME[0],
239 conf.DBPASSWD[0], db=conf.DBNAME[0])
236240 cursor = connection.cursor(oursql.DictCursor)
237 cursor.execute('select l.name, c.country_name from `location` as l, `codes` as c where l.name like ? and l.code=c.code limit ?', (term+'%', 5))
241 cursor.execute('select l.name, c.country_name from `location` as l, ' +
242 ' `codes` as c where l.name like ? and l.code=c.code' +
243 ' limit ?', (term+'%', 5))
238244 r = cursor.fetchall()
239245 connection.close()
240246 d = {}
248248 response = jsonify(d)
249249 response.headers['Access-Control-Allow-Origin'] = '*'
250250 return response
251
252
251253@app.route('/getLang', methods=['GET'])
252254def get_lang():
253255 term = request.args['term']
254 connection = oursql.Connection(conf.DBHOST[0],conf.DBUSRNAME[0],conf.DBPASSWD[0],db=conf.DBNAME[0])
256 connection = oursql.Connection(conf.DBHOST[0], conf.DBUSRNAME[0],
257 conf.DBPASSWD[0], db=conf.DBNAME[0])
255258 cursor = connection.cursor(oursql.DictCursor)
256 cursor.execute('select * from `languages` as l where l.name like ? limit ?', (term+'%',5))
259 cursor.execute('select * from `languages` as l where l.name like' +
260 ' ? limit ?', (term+'%', 5))
257261 r = cursor.fetchall()
258262 connection.close()
259263 d = {}
266266 response.headers['Access-Control-Allow-Origin'] = '*'
267267 return response
268268
269
269270@app.route('/blank', methods=['GET'])
270271def serve_blank():
271272 return render_template('blank.html')
272273
274
273275@app.route('/info', methods=['GET'])
274276def serve_info():
275277 coll = g.db['post']
276278 d = {}
277279 cntr = 0
278 for i in coll.find({"about":unquote_plus(request.args['about']),"lang":request.args['lang']}):
280 for i in coll.find({"about": unquote_plus(request.args['about']),
281 "lang": request.args['lang']}):
279282 i['_id'] = str(i['_id'])
280283 d[cntr] = i
281 cntr+=1
284 cntr += 1
282285 response = jsonify(d)
283286 response.headers['Access-Control-Allow-Origin'] = '*'
284287 return response
293293 lang = request.args['lang']
294294 url = request.args['url']
295295 query = collection.group(
296 key = Code('function(doc){return {"xpath" : doc.xpath, "about": doc.url}}'),
297 condition={"about" : url, "lang" : lang,"elementtype":"text"},
296 key=Code('function(doc){' +
297 'return {"xpath": doc.xpath, "about": doc.url}}'),
298 condition={"about": url, "lang": lang, "elementtype": "text"},
298299 initial={'narration': []},
299300 reduce=Code('function(doc,out){out.narration.push(doc);}')
300301 )
301302
302303 print query
303304
304 audio_query =collection.group(
305 key = Code('function(doc){return {"xpath" : doc.xpath, "about": doc.url}}'),
306 condition={"about" : url, "lang" : lang, 'elementtype':"audio/ogg"},
305 audio_query = collection.group(
306 key=Code('function(doc){' +
307 'return {"xpath": doc.xpath, "about": doc.url}}'),
308 condition={"about": url, "lang": lang, 'elementtype': "audio/ogg"},
307309 initial={'narration': []},
308310 reduce=Code('function(doc,out){out.narration.push(doc);}')
309311 )
310312
311 image_query =collection.group(
312 key = Code('function(doc){return {"xpath" : doc.xpath, "about": doc.url}}'),
313 condition={"about" : url, "lang" : lang, 'elementtype':"image"},
313 image_query = collection.group(
314 key=Code('function(doc){' +
315 'return {"xpath": doc.xpath, "about": doc.url}}'),
316 condition={"about": url, "lang": lang, 'elementtype': "image"},
314317 initial={'narration': []},
315318 reduce=Code('function(doc,out){out.narration.push(doc);}')
316319 )
337337 response.headers['Access-Control-Allow-Origin'] = '*'
338338 return response
339339
340
340341@app.route('/feeds', methods=['GET'])
341342def serve_feed_temp():
342343 return render_template("feeds.html")
343344
345
344346@app.route('/feed', methods=['GET'])
345347def serve_feed():
346348 coll = g.db['post']
347349 d = {}
348350 cntr = 0
349 for i in coll.find().sort('_id',direction=-1):
351 for i in coll.find().sort('_id', direction=-1):
350352 if i['data'] != '<br/>':
351353 i['_id'] = str(i['_id'])
352354 d[cntr] = i
353 cntr+=1
355 cntr += 1
354356 response = jsonify(d)
355357 response.headers['Access-Control-Allow-Origin'] = '*'
356358 return response
357359
360
358361@app.route('/about', methods=['GET'])
359362def serve_authors():
360363 coll = g.db['post']
361364 d = {}
362365 cntr = 0
363 for i in coll.find({"about":unquote_plus(request.args['about'])}):
366 for i in coll.find({"about": unquote_plus(request.args['about'])}):
364367 i['_id'] = str(i['_id'])
365368 d[cntr] = i
366 cntr+=1
369 cntr += 1
367370 response = jsonify(d)
368371 response.headers['Access-Control-Allow-Origin'] = '*'
369372 return response
373
374
370375#Retrieve all information about a specific $about and a given $author.
371376@app.route('/author', methods=['GET'])
372377def serve_author():
373378 coll = g.db['post']
374379 d = {}
375380 cntr = 0
376 for i in coll.find({"about":unquote_plus(request.args['about']),"author":unquote_plus(request.args['author'])}):
381 for i in coll.find({"about": unquote_plus(request.args['about']),
382 "author": unquote_plus(request.args['author'])}):
377383 i['_id'] = str(i['_id'])
378384 d[cntr] = i
379385 cntr += 1
387387 response.headers['Access-Control-Allow-Origin'] = '*'
388388 return response
389389
390
390391@app.route('/getAllLang', methods=['GET'])
391392def get_all_lang():
392393 term = request.args['term']
393 connection = oursql.Connection(conf.DBHOST[0],conf.DBUSRNAME[0],conf.DBPASSWD[0],db=conf.DBNAME[0])
394 connection = oursql.Connection(conf.DBHOST[0], conf.DBUSRNAME[0],
395 conf.DBPASSWD[0], db=conf.DBNAME[0])
394396 cursor = connection.cursor(oursql.DictCursor)
395 cursor.execute('select * from `languages` as l where l.name like ?', (term+'%',))
397 cursor.execute('select * from `languages` as l where l.name like ?',
398 (term+'%',))
396399 r = cursor.fetchall()
397400 connection.close()
398401 d = {}
410410 data = json.loads(request.form['data'])
411411 collection = g.db['post']
412412 page = {}
413 if type(data) is unicode: #A hack to fix malformed data. FIXME.
413 if type(data) is unicode: # A hack to fix malformed data. FIXME.
414414 data = json.loads(data)
415415 content = []
416 for i in data: #Create content objects here for posting to blog. DELETEME.
416 for i in data:
417 # Create content objects here for posting to blog. DELETEME.
417418 if 'comments' in i:
418419 page['comments'] = i['comments']
419420 else:
420421 contentobj = {}
421422 contentobj['type'] = i['elementtype']
422 contentobj['attr'] = {"language":i['lang'], "location":i['location'], "about":i['about'], "xpath":i['xpath']}
423 contentobj['attr'] = {"language": i['lang'],
424 "location": i['location'],
425 "about": i['about'],
426 "xpath": i['xpath']}
423427 contentobj['data'] = i['data']
424428 content.append(contentobj)
425429 i['bxpath'] = ''
433433 page['name'] = "About " + content[0]['attr']['about']
434434 page['content'] = content
435435
436 g.response_from_blogger = requests.api.post(conf.CUSTOM_BLOG_POST_URL[0], json.dumps(page), headers={"content-type":"application/json"})
436 g.response_from_blogger = requests.api.post(conf.CUSTOM_BLOG_POST_URL[0],
437 json.dumps(page),
438 headers={"content-type":
439 "application/json"})
437440 print "response from blogger " + repr(g.response_from_blogger)
438441 sweet(data)
439442 reply = make_response()
444444
445445
446446def sweet(data):
447 """ A function to sweet the data that is inserted. Accepts a <list of dicts>. """
447 """ A function to sweet the data that is inserted.
448 Accepts a <list of dicts>. """
448449 for i in data:
449450 if 'type' in i:
450451 del(i['_id'])
451 sweetmaker.sweet(conf.SWEET_STORE_ADD[0], [{"what":i['type'], "who":i['author'], "where":i['about']+i['xpath'], "how":conf.CUSTOM_BLOG_URL[0]+"/#"+g.response_from_blogger.json()['name']+' {lang: '+i["lang"]+',loc: '+i["location"]+'}'}])
452 sweetmaker.sweet(conf.SWEET_STORE_ADD[0],
453 [{"what": i['type'],
454 "who": i['author'],
455 "where":i['about']+i['xpath'],
456 "how":{'blogUrl': '{0}/#{1}'.format(
457 conf.CUSTOM_BLOG_URL[0],
458 g.response_from_blogger.json()['name']),
459 'language': i['lang'],
460 'location': i['location']}}])
452461 return True
453462 # data = json.dumps(data)
454463 # req = requests.api.post(conf.SWEETURL[0]+"/add",{'data':data})
479479 reply = make_response()
480480 return reply
481481
482@app.route("/menu",methods=['GET'])
482
483@app.route("/menu", methods=['GET'])
483484def menuForDialog():
484485 if request.args.has_key('option') == False:
485486 collection = g.db['post']
486487 c = {}
487488 cntr = 0
488489 print request.args['url']
489 for i in collection.find({"about":request.args['url']}).distinct('lang'):
490 for j in collection.find({"about":request.args['url'],'lang':i}).distinct('type'):
490 for i in collection.find({"about":
491 request.args['url']}).distinct('lang'):
492 for j in collection.find({"about": request.args['url'],
493 'lang': i}).distinct('type'):
491494 d = {}
492495 d['lang'] = i
493496 d['type'] = j
502502 collection = g.db['post']
503503 #get the ren languages for the received url
504504 langForUrl = collection.group(
505 key = Code('function(doc){return {"about" : doc.about}}'),
506 condition={"about" : d['url'],"blog":{'$regex':'/'+d['option']+'.*/'}},
505 key=Code('function(doc){return {"about" : doc.about}}'),
506 condition={"about": d['url'], "blog": {'$regex':
507 '/'+d['option']+'.*/'}},
507508 initial={'lang': []},
508 reduce=Code('function(doc, out){if (out.lang.indexOf(doc.lang) == -1) out.lang.push(doc.lang)}') #here xpath for test
509 reduce=Code('function(doc, out){' +
510 'if (out.lang.indexOf(doc.lang) == -1)' +
511 'out.lang.push(doc.lang)}') # here xpath for test
509512 )
510513
511514 #send the response
526526 url = request.args['url']
527527 #all re-narrations of the same xpath are grouped
528528 query = collection.group(
529 key = None,
530 condition={"about" :{'$regex':url+'*'}},
529 key=None,
530 condition={"about": {'$regex': url+'*'}},
531531 initial={'narration': []},
532532 reduce=Code('function(doc,out){out.narration.push(doc["about"]);}')
533533 )
534534
535 string=''
536 if len(query)==0:
537 return jsonify({'0':'empty'})
535 string = ''
536 if len(query) == 0:
537 return jsonify({'0': 'empty'})
538538 else:
539539 otherlist = {}
540540 cntr = -1
548548 return jsonify(otherlist)
549549
550550
551import logging,os
551import logging
552import os
552553from logging import FileHandler
553554
554fil = FileHandler(os.path.join(os.path.dirname(__file__),'logme'),mode='a')
555fil = FileHandler(os.path.join(os.path.dirname(__file__), 'logme'), mode='a')
555556fil.setLevel(logging.ERROR)
556557app.logger.addHandler(fil)
557558