tristantao / py-bing-search Goto Github PK
View Code? Open in Web Editor NEWPython Bing Search API
License: MIT License
Python Bing Search API
License: MIT License
Whenever, we try to execute this following command, using bing web search. We are getting this error.
second_fifty_result= bing_web.search(limit=1, format='json') #51-100
In the above code, bing_web.search
is not working. I got the following error.
`TimeoutError Traceback (most recent call last)
~\Anaconda3\lib\site-packages\urllib3\connection.py in _new_conn(self)
140 conn = connection.create_connection(
--> 141 (self.host, self.port), self.timeout, **extra_kw)
142
~\Anaconda3\lib\site-packages\urllib3\util\connection.py in create_connection(address, timeout, source_address, socket_options)
82 if err is not None:
---> 83 raise err
84
~\Anaconda3\lib\site-packages\urllib3\util\connection.py in create_connection(address, timeout, source_address, socket_options)
72 sock.bind(source_address)
---> 73 sock.connect(sa)
74 return sock
TimeoutError: [WinError 10060] A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond
During handling of the above exception, another exception occurred:
NewConnectionError Traceback (most recent call last)
~\Anaconda3\lib\site-packages\urllib3\connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
600 body=body, headers=headers,
--> 601 chunked=chunked)
602
~\Anaconda3\lib\site-packages\urllib3\connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
345 try:
--> 346 self._validate_conn(conn)
347 except (SocketTimeout, BaseSSLError) as e:
~\Anaconda3\lib\site-packages\urllib3\connectionpool.py in _validate_conn(self, conn)
849 if not getattr(conn, 'sock', None): # AppEngine might not have .sock
--> 850 conn.connect()
851
~\Anaconda3\lib\site-packages\urllib3\connection.py in connect(self)
283 # Add certificate verification
--> 284 conn = self._new_conn()
285
~\Anaconda3\lib\site-packages\urllib3\connection.py in _new_conn(self)
149 raise NewConnectionError(
--> 150 self, "Failed to establish a new connection: %s" % e)
151
NewConnectionError: <urllib3.connection.VerifiedHTTPSConnection object at 0x000001EF85486FD0>: Failed to establish a new connection: [WinError 10060] A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond
During handling of the above exception, another exception occurred:
MaxRetryError Traceback (most recent call last)
~\Anaconda3\lib\site-packages\requests\adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
439 retries=self.max_retries,
--> 440 timeout=timeout
441 )
~\Anaconda3\lib\site-packages\urllib3\connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
638 retries = retries.increment(method, url, error=e, _pool=self,
--> 639 _stacktrace=sys.exc_info()[2])
640 retries.sleep()
~\Anaconda3\lib\site-packages\urllib3\util\retry.py in increment(self, method, url, response, error, _pool, _stacktrace)
387 if new_retry.is_exhausted():
--> 388 raise MaxRetryError(_pool, url, error or ResponseError(cause))
389
MaxRetryError: HTTPSConnectionPool(host='api.datamarket.azure.com', port=443): Max retries exceeded with url: /Bing/Search/Web?Query=%27Python%20Software%20Foundation%27&$top=1&$skip=0&$format=json (Caused by NewConnectionError('<urllib3.connection.VerifiedHTTPSConnection object at 0x000001EF85486FD0>: Failed to establish a new connection: [WinError 10060] A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond',))
During handling of the above exception, another exception occurred:
ConnectionError Traceback (most recent call last)
in ()
1 # first_fifty_result= bing_web.search(limit=50, format='json')
2 #1-5
----> 3 second_fifty_result= bing_web.search(limit=1, format='json') #51-100
4 # print (second_fifty_result[0].description)
~\Anaconda3\lib\site-packages\py_bing_search\py_bing_search.py in search(self, limit, format)
18 def search(self, limit=50, format='json'):
19 ''' Returns the result list, and also the uri for next page (returned_list, next_uri) '''
---> 20 return self._search(limit, format)
21
22 def search_all(self, limit=50, format='json'):
~\Anaconda3\lib\site-packages\py_bing_search\py_bing_search.py in _search(self, limit, format)
61 '''
62 url = self.QUERY_URL.format(requests.utils.quote("'{}'".format(self.query)), min(50, limit), self.current_offset, format)
---> 63 r = requests.get(url, auth=("", self.api_key))
64 try:
65 json_results = r.json()
~\Anaconda3\lib\site-packages\requests\api.py in get(url, params, **kwargs)
70
71 kwargs.setdefault('allow_redirects', True)
---> 72 return request('get', url, params=params, **kwargs)
73
74
~\Anaconda3\lib\site-packages\requests\api.py in request(method, url, **kwargs)
56 # cases, and look like a memory leak in others.
57 with sessions.Session() as session:
---> 58 return session.request(method=method, url=url, **kwargs)
59
60
~\Anaconda3\lib\site-packages\requests\sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
506 }
507 send_kwargs.update(settings)
--> 508 resp = self.send(prep, **send_kwargs)
509
510 return resp
~\Anaconda3\lib\site-packages\requests\sessions.py in send(self, request, **kwargs)
616
617 # Send the request
--> 618 r = adapter.send(request, **kwargs)
619
620 # Total elapsed time of the request (approximately)
~\Anaconda3\lib\site-packages\requests\adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
506 raise SSLError(e, request=request)
507
--> 508 raise ConnectionError(e, request=request)
509
510 except ClosedPoolError as e:
ConnectionError: HTTPSConnectionPool(host='api.datamarket.azure.com', port=443): Max retries exceeded with url: /Bing/Search/Web?Query=%27Python%20Software%20Foundation%27&$top=1&$skip=0&$format=json (Caused by NewConnectionError('<urllib3.connection.VerifiedHTTPSConnection object at 0x000001EF85486FD0>: Failed to establish a new connection: [WinError 10060] A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond',))`
Hi,
I am getting this kind of error;
HTTPSConnectionPool(host='api.datamarket.azure.com', port=443): Max retries exceeded with url: /Bing/Search/Web?Query=%27Head%20%26%20Shoulders%27&$top=50&$skip=0&$format=json (Caused by NewConnectionError('<urllib3.connection.VerifiedHTTPSConnection object at 0x7f99163b7d68>: Failed to establish a new connection: [Errno 110] Connection timed out',))
I am not able to resolve it. :(
In [47]: result_list, next_uri = bing.search_all(kw, limit=30, format='json')
[14:22:04][INFO] requests.packages.urllib3.connectionpool connectionpool.py:_new_conn:756 | Starting new HTTPS connection (1): api.datamarket.azure.com
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-47-2750c83d88ac> in <module>()
----> 1 result_list, next_uri = bing.search_all(kw, limit=30, format='json')
/Users/erik/.virtualenvs/retargeting/lib/python2.7/site-packages/py_bing_search/py_bing_search.pyc in search_all(self, query, limit, format)
23 def search_all(self, query, limit=50, format='json'):
24 results = self._search(query, limit, 0, format)
---> 25 while results.total > len(results) and len(results) < limit:
26 max = limit - len(results)
27 more_results = self._search(query, max, len(results), format)
AttributeError: 'tuple' object has no attribute 'total'
Note that this is using bing.QUERY_URL = 'https://api.datamarket.azure.com/Bing/Search/v1/Web?Query={}&$top={}&$skip={}&$format={}'
-- the default SearchWeb
returns the following API error:
PyBingException: Request returned with code 403, error msg: <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> [823/1877]
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1"/>
<title>403 - Forbidden: Access is denied.</title>
<style type="text/css">
<!--
body{margin:0;font-size:.7em;font-family:Verdana, Arial, Helvetica, sans-serif;background:#EEEEEE;}
fieldset{padding:0 15px 10px 15px;}
h1{font-size:2.4em;margin:0;color:#FFF;}
h2{font-size:1.7em;margin:0;color:#CC0000;}
h3{font-size:1.2em;margin:10px 0 0 0;color:#000000;}
#header{width:96%;margin:0 0 0 0;padding:6px 2% 6px 2%;font-family:"trebuchet MS", Verdana, sans-serif;color:#FFF;
background-color:#555555;}
#content{margin:0 0 0 2%;position:relative;}
.content-container{background:#FFF;width:96%;margin-top:8px;padding:10px;position:relative;}
-->
</style>
</head>
<body>
<div id="header"><h1>Server Error</h1></div>
<div id="content">
<div class="content-container"><fieldset>
<h2>403 - Forbidden: Access is denied.</h2>
<h3>You do not have permission to view this directory or page using the credentials that you supplied.</h3>
</fieldset></div>
</div>
</body>
</html>
This is what I got:
py_bing_search.py_bing_search.PyBingImageException: Request returned with code 401, error msg: The authorization type you provided is not supported. Only Basic and OAuth are supported
Hi Tristantao,
Thank you so much for providing the wrapper which is helping us a lot. Wanted to know as Microsoft is migrating BingSearchApi to Microsoft Cognitive services, can we expect a new version of the wrapper code if so by when? Can you please look into it as most of my work depends on this wrapper and it would be a great help if you can fix this
Regards
Doesn't seem to hande Unicode characters too well..
# Fails
bing.search_all(u'óla', limit=50)
>>> UnicodeEncodeError: 'ascii' codec can't encode character u'\xf3' in position 0: ordinal not in range(128)
# Works
bing.search_all(u'óla'.encode('utf-8'), limit=50)
>>> [results]
Would it be good to add the encoding built-in to the code?
My suggestion would be to add to py_bing_search.py:37
a line:
query = query.encode('utf-8')
However I suck when it comes to unicode and the like so this might bring errors for other strings?
Currently it errors out with unable to detect JSON.
First, thank you for your work!
Is there a way to not specify the limit in search_all and just receive all results that Bing can provide?
Hi,
Can you give an example on how to search just news using this wrapper?
Thanks
Hi,
I will try to send a pull request today with the ability to choose whether safesearch is on or off (can be on by default, as bing supports)
but would be useful to have control over this param.
Hey, great library - thanks for doing this.
I'm having a bit of trouble with the auth. I'm sure I'm being a bonehead but I'm stuck.
I followed the directions and got an api key and created the cfg file.
Here's the error log:
$ python setup.py test
running test
running egg_info
writing requirements to py_bing_search.egg-info/requires.txt
writing py_bing_search.egg-info/PKG-INFO
writing top-level names to py_bing_search.egg-info/top_level.txt
writing dependency_links to py_bing_search.egg-info/dependency_links.txt
reading manifest file 'py_bing_search.egg-info/SOURCES.txt'
reading manifest template 'MANIFEST.in'
writing manifest file 'py_bing_search.egg-info/SOURCES.txt'
running build_ext
test_can_search (tests.test_py_bing_search.TestPyBingImageSearch) ... ERROR
test_search_all (tests.test_py_bing_search.TestPyBingImageSearch) ... ERROR
test_can_search (tests.test_py_bing_search.TestPyBingNewsSearch) ... ERROR
test_search_all (tests.test_py_bing_search.TestPyBingNewsSearch) ... ERROR
test_can_search (tests.test_py_bing_search.TestPyBingVideoSearch) ... ERROR
test_search_all (tests.test_py_bing_search.TestPyBingVideoSearch) ... ERROR
test_can_search (tests.test_py_bing_search.TestPyBingWebSearch) ... ERROR
test_search_all (tests.test_py_bing_search.TestPyBingWebSearch) ... ERROR
Traceback (most recent call last):
File "/Users/cuong_williams/Packages/py-bing-search/github/py-bing-search/tests/test_py_bing_search.py", line 48, in test_can_search
result_one = web_bing.search(limit=50)
File "/Users/cuong_williams/Packages/py-bing-search/github/py-bing-search/py_bing_search/py_bing_search.py", line 20, in search
return self._search(limit, format)
File "/Users/cuong_williams/Packages/py-bing-search/github/py-bing-search/py_bing_search/py_bing_search.py", line 149, in _search
raise PyBingImageException("Request returned with code %s, error msg: %s" % (r.status_code, r.text))
PyBingImageException: Request returned with code 401, error msg: The authorization type you provided is not supported. Only Basic and OAuth are supported
-------------------- >> begin captured logging << --------------------
requests.packages.urllib3.connectionpool: INFO: Starting new HTTPS connection (1): api.datamarket.azure.com
requests.packages.urllib3.connectionpool: DEBUG: "GET /Bing/Search/Image?Query=%27Python%20Software%20Foundation%27&$top=50&$skip=0&$format=json&ImageFilters=%27%27 HTTP/1.1" 401 91
--------------------- >> end captured logging << ---------------------
Traceback (most recent call last):
File "/Users/cuong_williams/Packages/py-bing-search/github/py-bing-search/tests/test_py_bing_search.py", line 54, in test_search_all
result_one = web_bing.search_all(limit=60)
File "/Users/cuong_williams/Packages/py-bing-search/github/py-bing-search/py_bing_search/py_bing_search.py", line 25, in search_all
results = self._search(limit, format)
File "/Users/cuong_williams/Packages/py-bing-search/github/py-bing-search/py_bing_search/py_bing_search.py", line 149, in _search
raise PyBingImageException("Request returned with code %s, error msg: %s" % (r.status_code, r.text))
PyBingImageException: Request returned with code 401, error msg: The authorization type you provided is not supported. Only Basic and OAuth are supported
-------------------- >> begin captured logging << --------------------
requests.packages.urllib3.connectionpool: INFO: Starting new HTTPS connection (1): api.datamarket.azure.com
requests.packages.urllib3.connectionpool: DEBUG: "GET /Bing/Search/Image?Query=%27Python%20Software%20Foundation%27&$top=50&$skip=0&$format=json&ImageFilters=%27%27 HTTP/1.1" 401 91
--------------------- >> end captured logging << ---------------------
Traceback (most recent call last):
File "/Users/cuong_williams/Packages/py-bing-search/github/py-bing-search/tests/test_py_bing_search.py", line 86, in test_can_search
result_one = web_bing.search(limit=50)
File "/Users/cuong_williams/Packages/py-bing-search/github/py-bing-search/py_bing_search/py_bing_search.py", line 20, in search
return self._search(limit, format)
File "/Users/cuong_williams/Packages/py-bing-search/github/py-bing-search/py_bing_search/py_bing_search.py", line 298, in _search
raise PyBingNewsException("Request returned with code %s, error msg: %s" % (r.status_code, r.text))
PyBingNewsException: Request returned with code 401, error msg: The authorization type you provided is not supported. Only Basic and OAuth are supported
-------------------- >> begin captured logging << --------------------
requests.packages.urllib3.connectionpool: INFO: Starting new HTTPS connection (1): api.datamarket.azure.com
requests.packages.urllib3.connectionpool: DEBUG: "GET /Bing/Search/News?Query=%27Python%20Software%20Foundation%27&$top=50&$skip=0&$format=json HTTP/1.1" 401 91
--------------------- >> end captured logging << ---------------------
Traceback (most recent call last):
File "/Users/cuong_williams/Packages/py-bing-search/github/py-bing-search/tests/test_py_bing_search.py", line 92, in test_search_all
result_one = web_bing.search_all(limit=60)
File "/Users/cuong_williams/Packages/py-bing-search/github/py-bing-search/py_bing_search/py_bing_search.py", line 25, in search_all
results = self._search(limit, format)
File "/Users/cuong_williams/Packages/py-bing-search/github/py-bing-search/py_bing_search/py_bing_search.py", line 298, in _search
raise PyBingNewsException("Request returned with code %s, error msg: %s" % (r.status_code, r.text))
PyBingNewsException: Request returned with code 401, error msg: The authorization type you provided is not supported. Only Basic and OAuth are supported
-------------------- >> begin captured logging << --------------------
requests.packages.urllib3.connectionpool: INFO: Starting new HTTPS connection (1): api.datamarket.azure.com
requests.packages.urllib3.connectionpool: DEBUG: "GET /Bing/Search/News?Query=%27Python%20Software%20Foundation%27&$top=50&$skip=0&$format=json HTTP/1.1" 401 91
--------------------- >> end captured logging << ---------------------
Traceback (most recent call last):
File "/Users/cuong_williams/Packages/py-bing-search/github/py-bing-search/tests/test_py_bing_search.py", line 67, in test_can_search
result_one = web_bing.search(limit=50)
File "/Users/cuong_williams/Packages/py-bing-search/github/py-bing-search/py_bing_search/py_bing_search.py", line 20, in search
return self._search(limit, format)
File "/Users/cuong_williams/Packages/py-bing-search/github/py-bing-search/py_bing_search/py_bing_search.py", line 226, in _search
raise PyBingVideoException("Request returned with code %s, error msg: %s" % (r.status_code, r.text))
PyBingVideoException: Request returned with code 401, error msg: The authorization type you provided is not supported. Only Basic and OAuth are supported
-------------------- >> begin captured logging << --------------------
requests.packages.urllib3.connectionpool: INFO: Starting new HTTPS connection (1): api.datamarket.azure.com
requests.packages.urllib3.connectionpool: DEBUG: "GET /Bing/Search/Video?Query=%27Python%20Software%20Foundation%27&$top=50&$skip=0&$format=json HTTP/1.1" 401 91
--------------------- >> end captured logging << ---------------------
Traceback (most recent call last):
File "/Users/cuong_williams/Packages/py-bing-search/github/py-bing-search/tests/test_py_bing_search.py", line 73, in test_search_all
result_one = web_bing.search_all(limit=60)
File "/Users/cuong_williams/Packages/py-bing-search/github/py-bing-search/py_bing_search/py_bing_search.py", line 25, in search_all
results = self._search(limit, format)
File "/Users/cuong_williams/Packages/py-bing-search/github/py-bing-search/py_bing_search/py_bing_search.py", line 226, in _search
raise PyBingVideoException("Request returned with code %s, error msg: %s" % (r.status_code, r.text))
PyBingVideoException: Request returned with code 401, error msg: The authorization type you provided is not supported. Only Basic and OAuth are supported
-------------------- >> begin captured logging << --------------------
requests.packages.urllib3.connectionpool: INFO: Starting new HTTPS connection (1): api.datamarket.azure.com
requests.packages.urllib3.connectionpool: DEBUG: "GET /Bing/Search/Video?Query=%27Python%20Software%20Foundation%27&$top=50&$skip=0&$format=json HTTP/1.1" 401 91
--------------------- >> end captured logging << ---------------------
Traceback (most recent call last):
File "/Users/cuong_williams/Packages/py-bing-search/github/py-bing-search/tests/test_py_bing_search.py", line 28, in test_can_search
result_one = web_bing.search(limit=50)
File "/Users/cuong_williams/Packages/py-bing-search/github/py-bing-search/py_bing_search/py_bing_search.py", line 20, in search
return self._search(limit, format)
File "/Users/cuong_williams/Packages/py-bing-search/github/py-bing-search/py_bing_search/py_bing_search.py", line 68, in _search
raise PyBingWebException("Request returned with code %s, error msg: %s" % (r.status_code, r.text))
PyBingWebException: Request returned with code 401, error msg: The authorization type you provided is not supported. Only Basic and OAuth are supported
-------------------- >> begin captured logging << --------------------
requests.packages.urllib3.connectionpool: INFO: Starting new HTTPS connection (1): api.datamarket.azure.com
requests.packages.urllib3.connectionpool: DEBUG: "GET /Bing/Search/Web?Query=%27Python%20Software%20Foundation%27&$top=50&$skip=0&$format=json HTTP/1.1" 401 91
--------------------- >> end captured logging << ---------------------
Traceback (most recent call last):
File "/Users/cuong_williams/Packages/py-bing-search/github/py-bing-search/tests/test_py_bing_search.py", line 35, in test_search_all
result_one = web_bing.search_all(limit=60)
File "/Users/cuong_williams/Packages/py-bing-search/github/py-bing-search/py_bing_search/py_bing_search.py", line 25, in search_all
results = self._search(limit, format)
File "/Users/cuong_williams/Packages/py-bing-search/github/py-bing-search/py_bing_search/py_bing_search.py", line 68, in _search
raise PyBingWebException("Request returned with code %s, error msg: %s" % (r.status_code, r.text))
PyBingWebException: Request returned with code 401, error msg: The authorization type you provided is not supported. Only Basic and OAuth are supported
-------------------- >> begin captured logging << --------------------
requests.packages.urllib3.connectionpool: INFO: Starting new HTTPS connection (1): api.datamarket.azure.com
requests.packages.urllib3.connectionpool: DEBUG: "GET /Bing/Search/Web?Query=%27Python%20Software%20Foundation%27&$top=50&$skip=0&$format=json HTTP/1.1" 401 91
--------------------- >> end captured logging << ---------------------
Ran 8 tests in 12.605s
FAILED (errors=8)
Using the first example on Python 3.5 I get the following error:
ImportError: cannot import name 'PyBingWebSearch'
I see there's already a pull request with appropriate fixes. When is it going to be merged into the main branch?
I think it should be: PyBingNewsSearch instead of PyBingNewsbSearch
In your example code...
File "BuildCallSrv.py", line 109, in NextResult
self._Result = self._Service.search_all(self._Query)
File "/usr/lib/python2.7/site-packages/py_bing_search/py_bing_search.py", line 25, in search_all
while results.total > len(results) and len(results) < limit:
AttributeError: 'tuple' object has no attribute 'total'
A declarative, efficient, and flexible JavaScript library for building user interfaces.
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
An Open Source Machine Learning Framework for Everyone
The Web framework for perfectionists with deadlines.
A PHP framework for web artisans
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
Some thing interesting about web. New door for the world.
A server is a program made to process requests and deliver data to clients.
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
Some thing interesting about visualization, use data art
Some thing interesting about game, make everyone happy.
We are working to build community through open source technology. NB: members must have two-factor auth.
Open source projects and samples from Microsoft.
Google ❤️ Open Source for everyone.
Alibaba Open Source for everyone
Data-Driven Documents codes.
China tencent open source team.